@vertesia/workflow 0.51.0 → 0.54.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (387) hide show
  1. package/package.json +9 -6
  2. package/src/activities/advanced/createOrUpdateDocumentFromInteractionRun.ts +20 -1
  3. package/src/activities/chunkDocument.ts +62 -42
  4. package/src/activities/createDocumentFromOther.ts +2 -2
  5. package/src/activities/executeInteraction.ts +92 -47
  6. package/src/activities/extractDocumentText.ts +91 -54
  7. package/src/activities/generateDocumentProperties.ts +37 -16
  8. package/src/activities/generateEmbeddings.ts +91 -79
  9. package/src/activities/generateImageRendition.ts +127 -59
  10. package/src/activities/generateOrAssignContentType.ts +52 -32
  11. package/src/activities/getObjectFromStore.ts +1 -1
  12. package/src/activities/handleError.ts +25 -0
  13. package/src/activities/index-dsl.ts +1 -0
  14. package/src/activities/index.ts +0 -1
  15. package/src/activities/media/processPdfWithTextract.ts +4 -4
  16. package/src/activities/media/transcribeMediaWithGladia.ts +1 -1
  17. package/src/activities/notifyWebhook.ts +2 -2
  18. package/src/activities/setDocumentStatus.ts +1 -1
  19. package/src/conversion/TextractProcessor.ts +9 -9
  20. package/src/conversion/image.test.ts +110 -18
  21. package/src/conversion/image.ts +96 -15
  22. package/src/conversion/markitdown.ts +41 -0
  23. package/src/conversion/mutool.ts +1 -1
  24. package/src/conversion/pandoc.test.ts +8 -6
  25. package/src/conversion/pandoc.ts +38 -42
  26. package/src/dsl/dsl-workflow.ts +80 -12
  27. package/src/dsl/setup/ActivityContext.ts +57 -16
  28. package/src/dsl/validation.test.ts +2 -2
  29. package/src/dsl/vars.test.ts +1 -1
  30. package/src/dsl/vars.ts +6 -6
  31. package/src/dsl/workflow-exec-child.test.ts +14 -4
  32. package/src/dsl/workflow-fetch.test.ts +1 -1
  33. package/src/dsl/workflow-import.test.ts +1 -1
  34. package/src/dsl/workflow.test.ts +12 -2
  35. package/src/dsl.ts +1 -1
  36. package/src/errors.ts +27 -6
  37. package/src/index.ts +1 -1
  38. package/src/iterative-generation/activities/extractToc.ts +1 -1
  39. package/src/iterative-generation/activities/generatePart.ts +2 -2
  40. package/src/iterative-generation/activities/generateToc.ts +1 -1
  41. package/src/iterative-generation/iterativeGenerationWorkflow.ts +3 -2
  42. package/src/iterative-generation/types.ts +4 -4
  43. package/src/iterative-generation/utils.ts +4 -4
  44. package/src/system/notifyWebhookWorkflow.ts +2 -1
  45. package/src/system/recalculateEmbeddingsWorkflow.ts +2 -2
  46. package/src/utils/blobs.ts +11 -6
  47. package/src/utils/chunks.ts +17 -0
  48. package/src/utils/client.ts +4 -3
  49. package/src/utils/memory.ts +3 -8
  50. package/lib/cjs/activities/advanced/createDocumentTypeFromInteractionRun.js +0 -32
  51. package/lib/cjs/activities/advanced/createDocumentTypeFromInteractionRun.js.map +0 -1
  52. package/lib/cjs/activities/advanced/createOrUpdateDocumentFromInteractionRun.js +0 -66
  53. package/lib/cjs/activities/advanced/createOrUpdateDocumentFromInteractionRun.js.map +0 -1
  54. package/lib/cjs/activities/advanced/updateDocumentFromInteractionRun.js +0 -18
  55. package/lib/cjs/activities/advanced/updateDocumentFromInteractionRun.js.map +0 -1
  56. package/lib/cjs/activities/chunkDocument.js +0 -79
  57. package/lib/cjs/activities/chunkDocument.js.map +0 -1
  58. package/lib/cjs/activities/createDocumentFromOther.js +0 -64
  59. package/lib/cjs/activities/createDocumentFromOther.js.map +0 -1
  60. package/lib/cjs/activities/executeInteraction.js +0 -134
  61. package/lib/cjs/activities/executeInteraction.js.map +0 -1
  62. package/lib/cjs/activities/extractDocumentText.js +0 -135
  63. package/lib/cjs/activities/extractDocumentText.js.map +0 -1
  64. package/lib/cjs/activities/generateDocumentProperties.js +0 -59
  65. package/lib/cjs/activities/generateDocumentProperties.js.map +0 -1
  66. package/lib/cjs/activities/generateEmbeddings.js +0 -292
  67. package/lib/cjs/activities/generateEmbeddings.js.map +0 -1
  68. package/lib/cjs/activities/generateImageRendition.js +0 -104
  69. package/lib/cjs/activities/generateImageRendition.js.map +0 -1
  70. package/lib/cjs/activities/generateOrAssignContentType.js +0 -103
  71. package/lib/cjs/activities/generateOrAssignContentType.js.map +0 -1
  72. package/lib/cjs/activities/getObjectFromStore.js +0 -20
  73. package/lib/cjs/activities/getObjectFromStore.js.map +0 -1
  74. package/lib/cjs/activities/index-dsl.js +0 -37
  75. package/lib/cjs/activities/index-dsl.js.map +0 -1
  76. package/lib/cjs/activities/index.js +0 -22
  77. package/lib/cjs/activities/index.js.map +0 -1
  78. package/lib/cjs/activities/media/processPdfWithTextract.js +0 -102
  79. package/lib/cjs/activities/media/processPdfWithTextract.js.map +0 -1
  80. package/lib/cjs/activities/media/transcribeMediaWithGladia.js +0 -51
  81. package/lib/cjs/activities/media/transcribeMediaWithGladia.js.map +0 -1
  82. package/lib/cjs/activities/notifyWebhook.js +0 -34
  83. package/lib/cjs/activities/notifyWebhook.js.map +0 -1
  84. package/lib/cjs/activities/setDocumentStatus.js +0 -15
  85. package/lib/cjs/activities/setDocumentStatus.js.map +0 -1
  86. package/lib/cjs/conversion/TextractProcessor.js +0 -417
  87. package/lib/cjs/conversion/TextractProcessor.js.map +0 -1
  88. package/lib/cjs/conversion/image.js +0 -22
  89. package/lib/cjs/conversion/image.js.map +0 -1
  90. package/lib/cjs/conversion/mutool.js +0 -147
  91. package/lib/cjs/conversion/mutool.js.map +0 -1
  92. package/lib/cjs/conversion/pandoc.js +0 -39
  93. package/lib/cjs/conversion/pandoc.js.map +0 -1
  94. package/lib/cjs/dsl/conditions.js +0 -81
  95. package/lib/cjs/dsl/conditions.js.map +0 -1
  96. package/lib/cjs/dsl/dsl-workflow.js +0 -223
  97. package/lib/cjs/dsl/dsl-workflow.js.map +0 -1
  98. package/lib/cjs/dsl/dslProxyActivities.js +0 -23
  99. package/lib/cjs/dsl/dslProxyActivities.js.map +0 -1
  100. package/lib/cjs/dsl/projections.js +0 -59
  101. package/lib/cjs/dsl/projections.js.map +0 -1
  102. package/lib/cjs/dsl/setup/ActivityContext.js +0 -96
  103. package/lib/cjs/dsl/setup/ActivityContext.js.map +0 -1
  104. package/lib/cjs/dsl/setup/fetch/DataProvider.js +0 -51
  105. package/lib/cjs/dsl/setup/fetch/DataProvider.js.map +0 -1
  106. package/lib/cjs/dsl/setup/fetch/index.js +0 -16
  107. package/lib/cjs/dsl/setup/fetch/index.js.map +0 -1
  108. package/lib/cjs/dsl/setup/fetch/providers.js +0 -67
  109. package/lib/cjs/dsl/setup/fetch/providers.js.map +0 -1
  110. package/lib/cjs/dsl/test/test-child-workflow.js +0 -10
  111. package/lib/cjs/dsl/test/test-child-workflow.js.map +0 -1
  112. package/lib/cjs/dsl/validation.js +0 -122
  113. package/lib/cjs/dsl/validation.js.map +0 -1
  114. package/lib/cjs/dsl/vars.js +0 -341
  115. package/lib/cjs/dsl/vars.js.map +0 -1
  116. package/lib/cjs/dsl/walk.js +0 -100
  117. package/lib/cjs/dsl/walk.js.map +0 -1
  118. package/lib/cjs/dsl.js +0 -20
  119. package/lib/cjs/dsl.js.map +0 -1
  120. package/lib/cjs/errors.js +0 -36
  121. package/lib/cjs/errors.js.map +0 -1
  122. package/lib/cjs/index.js +0 -50
  123. package/lib/cjs/index.js.map +0 -1
  124. package/lib/cjs/iterative-generation/activities/extractToc.js +0 -47
  125. package/lib/cjs/iterative-generation/activities/extractToc.js.map +0 -1
  126. package/lib/cjs/iterative-generation/activities/finalizeOutput.js +0 -69
  127. package/lib/cjs/iterative-generation/activities/finalizeOutput.js.map +0 -1
  128. package/lib/cjs/iterative-generation/activities/generatePart.js +0 -73
  129. package/lib/cjs/iterative-generation/activities/generatePart.js.map +0 -1
  130. package/lib/cjs/iterative-generation/activities/generateToc.js +0 -91
  131. package/lib/cjs/iterative-generation/activities/generateToc.js.map +0 -1
  132. package/lib/cjs/iterative-generation/activities/index.js +0 -12
  133. package/lib/cjs/iterative-generation/activities/index.js.map +0 -1
  134. package/lib/cjs/iterative-generation/iterativeGenerationWorkflow.js +0 -55
  135. package/lib/cjs/iterative-generation/iterativeGenerationWorkflow.js.map +0 -1
  136. package/lib/cjs/iterative-generation/types.js +0 -5
  137. package/lib/cjs/iterative-generation/types.js.map +0 -1
  138. package/lib/cjs/iterative-generation/utils.js +0 -121
  139. package/lib/cjs/iterative-generation/utils.js.map +0 -1
  140. package/lib/cjs/package.json +0 -3
  141. package/lib/cjs/result-types.js +0 -10
  142. package/lib/cjs/result-types.js.map +0 -1
  143. package/lib/cjs/system/notifyWebhookWorkflow.js +0 -46
  144. package/lib/cjs/system/notifyWebhookWorkflow.js.map +0 -1
  145. package/lib/cjs/system/recalculateEmbeddingsWorkflow.js +0 -28
  146. package/lib/cjs/system/recalculateEmbeddingsWorkflow.js.map +0 -1
  147. package/lib/cjs/utils/auth.js +0 -15
  148. package/lib/cjs/utils/auth.js.map +0 -1
  149. package/lib/cjs/utils/blobs.js +0 -63
  150. package/lib/cjs/utils/blobs.js.map +0 -1
  151. package/lib/cjs/utils/client.js +0 -25
  152. package/lib/cjs/utils/client.js.map +0 -1
  153. package/lib/cjs/utils/expand-vars.js +0 -33
  154. package/lib/cjs/utils/expand-vars.js.map +0 -1
  155. package/lib/cjs/utils/memory.js +0 -72
  156. package/lib/cjs/utils/memory.js.map +0 -1
  157. package/lib/cjs/utils/tokens.js +0 -38
  158. package/lib/cjs/utils/tokens.js.map +0 -1
  159. package/lib/cjs/vars.js +0 -20
  160. package/lib/cjs/vars.js.map +0 -1
  161. package/lib/cjs/workflows.js +0 -15
  162. package/lib/cjs/workflows.js.map +0 -1
  163. package/lib/esm/activities/advanced/createDocumentTypeFromInteractionRun.js +0 -29
  164. package/lib/esm/activities/advanced/createDocumentTypeFromInteractionRun.js.map +0 -1
  165. package/lib/esm/activities/advanced/createOrUpdateDocumentFromInteractionRun.js +0 -63
  166. package/lib/esm/activities/advanced/createOrUpdateDocumentFromInteractionRun.js.map +0 -1
  167. package/lib/esm/activities/advanced/updateDocumentFromInteractionRun.js +0 -15
  168. package/lib/esm/activities/advanced/updateDocumentFromInteractionRun.js.map +0 -1
  169. package/lib/esm/activities/chunkDocument.js +0 -76
  170. package/lib/esm/activities/chunkDocument.js.map +0 -1
  171. package/lib/esm/activities/createDocumentFromOther.js +0 -58
  172. package/lib/esm/activities/createDocumentFromOther.js.map +0 -1
  173. package/lib/esm/activities/executeInteraction.js +0 -130
  174. package/lib/esm/activities/executeInteraction.js.map +0 -1
  175. package/lib/esm/activities/extractDocumentText.js +0 -132
  176. package/lib/esm/activities/extractDocumentText.js.map +0 -1
  177. package/lib/esm/activities/generateDocumentProperties.js +0 -56
  178. package/lib/esm/activities/generateDocumentProperties.js.map +0 -1
  179. package/lib/esm/activities/generateEmbeddings.js +0 -256
  180. package/lib/esm/activities/generateEmbeddings.js.map +0 -1
  181. package/lib/esm/activities/generateImageRendition.js +0 -98
  182. package/lib/esm/activities/generateImageRendition.js.map +0 -1
  183. package/lib/esm/activities/generateOrAssignContentType.js +0 -100
  184. package/lib/esm/activities/generateOrAssignContentType.js.map +0 -1
  185. package/lib/esm/activities/getObjectFromStore.js +0 -17
  186. package/lib/esm/activities/getObjectFromStore.js.map +0 -1
  187. package/lib/esm/activities/index-dsl.js +0 -18
  188. package/lib/esm/activities/index-dsl.js.map +0 -1
  189. package/lib/esm/activities/index.js +0 -6
  190. package/lib/esm/activities/index.js.map +0 -1
  191. package/lib/esm/activities/media/processPdfWithTextract.js +0 -98
  192. package/lib/esm/activities/media/processPdfWithTextract.js.map +0 -1
  193. package/lib/esm/activities/media/transcribeMediaWithGladia.js +0 -48
  194. package/lib/esm/activities/media/transcribeMediaWithGladia.js.map +0 -1
  195. package/lib/esm/activities/notifyWebhook.js +0 -31
  196. package/lib/esm/activities/notifyWebhook.js.map +0 -1
  197. package/lib/esm/activities/setDocumentStatus.js +0 -12
  198. package/lib/esm/activities/setDocumentStatus.js.map +0 -1
  199. package/lib/esm/conversion/TextractProcessor.js +0 -410
  200. package/lib/esm/conversion/TextractProcessor.js.map +0 -1
  201. package/lib/esm/conversion/image.js +0 -16
  202. package/lib/esm/conversion/image.js.map +0 -1
  203. package/lib/esm/conversion/mutool.js +0 -139
  204. package/lib/esm/conversion/mutool.js.map +0 -1
  205. package/lib/esm/conversion/pandoc.js +0 -36
  206. package/lib/esm/conversion/pandoc.js.map +0 -1
  207. package/lib/esm/dsl/conditions.js +0 -75
  208. package/lib/esm/dsl/conditions.js.map +0 -1
  209. package/lib/esm/dsl/dsl-workflow.js +0 -216
  210. package/lib/esm/dsl/dsl-workflow.js.map +0 -1
  211. package/lib/esm/dsl/dslProxyActivities.js +0 -20
  212. package/lib/esm/dsl/dslProxyActivities.js.map +0 -1
  213. package/lib/esm/dsl/projections.js +0 -55
  214. package/lib/esm/dsl/projections.js.map +0 -1
  215. package/lib/esm/dsl/setup/ActivityContext.js +0 -91
  216. package/lib/esm/dsl/setup/ActivityContext.js.map +0 -1
  217. package/lib/esm/dsl/setup/fetch/DataProvider.js +0 -47
  218. package/lib/esm/dsl/setup/fetch/DataProvider.js.map +0 -1
  219. package/lib/esm/dsl/setup/fetch/index.js +0 -12
  220. package/lib/esm/dsl/setup/fetch/index.js.map +0 -1
  221. package/lib/esm/dsl/setup/fetch/providers.js +0 -61
  222. package/lib/esm/dsl/setup/fetch/providers.js.map +0 -1
  223. package/lib/esm/dsl/test/test-child-workflow.js +0 -5
  224. package/lib/esm/dsl/test/test-child-workflow.js.map +0 -1
  225. package/lib/esm/dsl/validation.js +0 -118
  226. package/lib/esm/dsl/validation.js.map +0 -1
  227. package/lib/esm/dsl/vars.js +0 -335
  228. package/lib/esm/dsl/vars.js.map +0 -1
  229. package/lib/esm/dsl/walk.js +0 -96
  230. package/lib/esm/dsl/walk.js.map +0 -1
  231. package/lib/esm/dsl.js +0 -4
  232. package/lib/esm/dsl.js.map +0 -1
  233. package/lib/esm/errors.js +0 -30
  234. package/lib/esm/errors.js.map +0 -1
  235. package/lib/esm/index.js +0 -32
  236. package/lib/esm/index.js.map +0 -1
  237. package/lib/esm/iterative-generation/activities/extractToc.js +0 -44
  238. package/lib/esm/iterative-generation/activities/extractToc.js.map +0 -1
  239. package/lib/esm/iterative-generation/activities/finalizeOutput.js +0 -66
  240. package/lib/esm/iterative-generation/activities/finalizeOutput.js.map +0 -1
  241. package/lib/esm/iterative-generation/activities/generatePart.js +0 -70
  242. package/lib/esm/iterative-generation/activities/generatePart.js.map +0 -1
  243. package/lib/esm/iterative-generation/activities/generateToc.js +0 -88
  244. package/lib/esm/iterative-generation/activities/generateToc.js.map +0 -1
  245. package/lib/esm/iterative-generation/activities/index.js +0 -5
  246. package/lib/esm/iterative-generation/activities/index.js.map +0 -1
  247. package/lib/esm/iterative-generation/iterativeGenerationWorkflow.js +0 -52
  248. package/lib/esm/iterative-generation/iterativeGenerationWorkflow.js.map +0 -1
  249. package/lib/esm/iterative-generation/types.js +0 -2
  250. package/lib/esm/iterative-generation/types.js.map +0 -1
  251. package/lib/esm/iterative-generation/utils.js +0 -112
  252. package/lib/esm/iterative-generation/utils.js.map +0 -1
  253. package/lib/esm/result-types.js +0 -7
  254. package/lib/esm/result-types.js.map +0 -1
  255. package/lib/esm/system/notifyWebhookWorkflow.js +0 -43
  256. package/lib/esm/system/notifyWebhookWorkflow.js.map +0 -1
  257. package/lib/esm/system/recalculateEmbeddingsWorkflow.js +0 -25
  258. package/lib/esm/system/recalculateEmbeddingsWorkflow.js.map +0 -1
  259. package/lib/esm/utils/auth.js +0 -8
  260. package/lib/esm/utils/auth.js.map +0 -1
  261. package/lib/esm/utils/blobs.js +0 -52
  262. package/lib/esm/utils/blobs.js.map +0 -1
  263. package/lib/esm/utils/client.js +0 -22
  264. package/lib/esm/utils/client.js.map +0 -1
  265. package/lib/esm/utils/expand-vars.js +0 -30
  266. package/lib/esm/utils/expand-vars.js.map +0 -1
  267. package/lib/esm/utils/memory.js +0 -60
  268. package/lib/esm/utils/memory.js.map +0 -1
  269. package/lib/esm/utils/tokens.js +0 -34
  270. package/lib/esm/utils/tokens.js.map +0 -1
  271. package/lib/esm/vars.js +0 -4
  272. package/lib/esm/vars.js.map +0 -1
  273. package/lib/esm/workflows.js +0 -8
  274. package/lib/esm/workflows.js.map +0 -1
  275. package/lib/types/activities/advanced/createDocumentTypeFromInteractionRun.d.ts +0 -17
  276. package/lib/types/activities/advanced/createDocumentTypeFromInteractionRun.d.ts.map +0 -1
  277. package/lib/types/activities/advanced/createOrUpdateDocumentFromInteractionRun.d.ts +0 -29
  278. package/lib/types/activities/advanced/createOrUpdateDocumentFromInteractionRun.d.ts.map +0 -1
  279. package/lib/types/activities/advanced/updateDocumentFromInteractionRun.d.ts +0 -19
  280. package/lib/types/activities/advanced/updateDocumentFromInteractionRun.d.ts.map +0 -1
  281. package/lib/types/activities/chunkDocument.d.ts +0 -18
  282. package/lib/types/activities/chunkDocument.d.ts.map +0 -1
  283. package/lib/types/activities/createDocumentFromOther.d.ts +0 -21
  284. package/lib/types/activities/createDocumentFromOther.d.ts.map +0 -1
  285. package/lib/types/activities/executeInteraction.d.ts +0 -44
  286. package/lib/types/activities/executeInteraction.d.ts.map +0 -1
  287. package/lib/types/activities/extractDocumentText.d.ts +0 -10
  288. package/lib/types/activities/extractDocumentText.d.ts.map +0 -1
  289. package/lib/types/activities/generateDocumentProperties.d.ts +0 -32
  290. package/lib/types/activities/generateDocumentProperties.d.ts.map +0 -1
  291. package/lib/types/activities/generateEmbeddings.d.ts +0 -49
  292. package/lib/types/activities/generateEmbeddings.d.ts.map +0 -1
  293. package/lib/types/activities/generateImageRendition.d.ts +0 -17
  294. package/lib/types/activities/generateImageRendition.d.ts.map +0 -1
  295. package/lib/types/activities/generateOrAssignContentType.d.ts +0 -44
  296. package/lib/types/activities/generateOrAssignContentType.d.ts.map +0 -1
  297. package/lib/types/activities/getObjectFromStore.d.ts +0 -14
  298. package/lib/types/activities/getObjectFromStore.d.ts.map +0 -1
  299. package/lib/types/activities/index-dsl.d.ts +0 -17
  300. package/lib/types/activities/index-dsl.d.ts.map +0 -1
  301. package/lib/types/activities/index.d.ts +0 -6
  302. package/lib/types/activities/index.d.ts.map +0 -1
  303. package/lib/types/activities/media/processPdfWithTextract.d.ts +0 -26
  304. package/lib/types/activities/media/processPdfWithTextract.d.ts.map +0 -1
  305. package/lib/types/activities/media/transcribeMediaWithGladia.d.ts +0 -14
  306. package/lib/types/activities/media/transcribeMediaWithGladia.d.ts.map +0 -1
  307. package/lib/types/activities/notifyWebhook.d.ts +0 -17
  308. package/lib/types/activities/notifyWebhook.d.ts.map +0 -1
  309. package/lib/types/activities/setDocumentStatus.d.ts +0 -15
  310. package/lib/types/activities/setDocumentStatus.d.ts.map +0 -1
  311. package/lib/types/conversion/TextractProcessor.d.ts +0 -45
  312. package/lib/types/conversion/TextractProcessor.d.ts.map +0 -1
  313. package/lib/types/conversion/image.d.ts +0 -9
  314. package/lib/types/conversion/image.d.ts.map +0 -1
  315. package/lib/types/conversion/mutool.d.ts +0 -19
  316. package/lib/types/conversion/mutool.d.ts.map +0 -1
  317. package/lib/types/conversion/pandoc.d.ts +0 -2
  318. package/lib/types/conversion/pandoc.d.ts.map +0 -1
  319. package/lib/types/dsl/conditions.d.ts +0 -2
  320. package/lib/types/dsl/conditions.d.ts.map +0 -1
  321. package/lib/types/dsl/dsl-workflow.d.ts +0 -5
  322. package/lib/types/dsl/dsl-workflow.d.ts.map +0 -1
  323. package/lib/types/dsl/dslProxyActivities.d.ts +0 -10
  324. package/lib/types/dsl/dslProxyActivities.d.ts.map +0 -1
  325. package/lib/types/dsl/projections.d.ts +0 -4
  326. package/lib/types/dsl/projections.d.ts.map +0 -1
  327. package/lib/types/dsl/setup/ActivityContext.d.ts +0 -14
  328. package/lib/types/dsl/setup/ActivityContext.d.ts.map +0 -1
  329. package/lib/types/dsl/setup/fetch/DataProvider.d.ts +0 -9
  330. package/lib/types/dsl/setup/fetch/DataProvider.d.ts.map +0 -1
  331. package/lib/types/dsl/setup/fetch/index.d.ts +0 -6
  332. package/lib/types/dsl/setup/fetch/index.d.ts.map +0 -1
  333. package/lib/types/dsl/setup/fetch/providers.d.ts +0 -25
  334. package/lib/types/dsl/setup/fetch/providers.d.ts.map +0 -1
  335. package/lib/types/dsl/test/test-child-workflow.d.ts +0 -4
  336. package/lib/types/dsl/test/test-child-workflow.d.ts.map +0 -1
  337. package/lib/types/dsl/validation.d.ts +0 -4
  338. package/lib/types/dsl/validation.d.ts.map +0 -1
  339. package/lib/types/dsl/vars.d.ts +0 -48
  340. package/lib/types/dsl/vars.d.ts.map +0 -1
  341. package/lib/types/dsl/walk.d.ts +0 -18
  342. package/lib/types/dsl/walk.d.ts.map +0 -1
  343. package/lib/types/dsl.d.ts +0 -4
  344. package/lib/types/dsl.d.ts.map +0 -1
  345. package/lib/types/errors.d.ts +0 -16
  346. package/lib/types/errors.d.ts.map +0 -1
  347. package/lib/types/index.d.ts +0 -31
  348. package/lib/types/index.d.ts.map +0 -1
  349. package/lib/types/iterative-generation/activities/extractToc.d.ts +0 -10
  350. package/lib/types/iterative-generation/activities/extractToc.d.ts.map +0 -1
  351. package/lib/types/iterative-generation/activities/finalizeOutput.d.ts +0 -3
  352. package/lib/types/iterative-generation/activities/finalizeOutput.d.ts.map +0 -1
  353. package/lib/types/iterative-generation/activities/generatePart.d.ts +0 -3
  354. package/lib/types/iterative-generation/activities/generatePart.d.ts.map +0 -1
  355. package/lib/types/iterative-generation/activities/generateToc.d.ts +0 -4
  356. package/lib/types/iterative-generation/activities/generateToc.d.ts.map +0 -1
  357. package/lib/types/iterative-generation/activities/index.d.ts +0 -5
  358. package/lib/types/iterative-generation/activities/index.d.ts.map +0 -1
  359. package/lib/types/iterative-generation/iterativeGenerationWorkflow.d.ts +0 -3
  360. package/lib/types/iterative-generation/iterativeGenerationWorkflow.d.ts.map +0 -1
  361. package/lib/types/iterative-generation/types.d.ts +0 -79
  362. package/lib/types/iterative-generation/types.d.ts.map +0 -1
  363. package/lib/types/iterative-generation/utils.d.ts +0 -27
  364. package/lib/types/iterative-generation/utils.d.ts.map +0 -1
  365. package/lib/types/result-types.d.ts +0 -22
  366. package/lib/types/result-types.d.ts.map +0 -1
  367. package/lib/types/system/notifyWebhookWorkflow.d.ts +0 -3
  368. package/lib/types/system/notifyWebhookWorkflow.d.ts.map +0 -1
  369. package/lib/types/system/recalculateEmbeddingsWorkflow.d.ts +0 -40
  370. package/lib/types/system/recalculateEmbeddingsWorkflow.d.ts.map +0 -1
  371. package/lib/types/utils/auth.d.ts +0 -4
  372. package/lib/types/utils/auth.d.ts.map +0 -1
  373. package/lib/types/utils/blobs.d.ts +0 -8
  374. package/lib/types/utils/blobs.d.ts.map +0 -1
  375. package/lib/types/utils/client.d.ts +0 -7
  376. package/lib/types/utils/client.d.ts.map +0 -1
  377. package/lib/types/utils/expand-vars.d.ts +0 -8
  378. package/lib/types/utils/expand-vars.d.ts.map +0 -1
  379. package/lib/types/utils/memory.d.ts +0 -12
  380. package/lib/types/utils/memory.d.ts.map +0 -1
  381. package/lib/types/utils/tokens.d.ts +0 -11
  382. package/lib/types/utils/tokens.d.ts.map +0 -1
  383. package/lib/types/vars.d.ts +0 -3
  384. package/lib/types/vars.d.ts.map +0 -1
  385. package/lib/types/workflows.d.ts +0 -8
  386. package/lib/types/workflows.d.ts.map +0 -1
  387. package/lib/workflows-bundle.js +0 -19897
@@ -1,9 +1,9 @@
1
- import { VertesiaClient, StreamSource } from "@vertesia/client";
1
+ import { VertesiaClient } from "@vertesia/client";
2
+ import { NodeStreamSource } from "@vertesia/client/node";
2
3
  import { Commands, MemoryPack, buildMemoryPack as _buildMemoryPack, loadMemoryPack as _loadMemoryPack } from "@vertesia/memory";
3
4
  import { createReadStream, createWriteStream } from "fs";
4
5
  import { rm } from "fs/promises";
5
- import { readableToWebStream, webStreamToReadable } from "node-web-stream-adapters";
6
- import { Readable } from "stream";
6
+ import { webStreamToReadable } from "node-web-stream-adapters";
7
7
  import { pipeline } from "stream/promises";
8
8
 
9
9
  import tmp from "tmp";
@@ -11,11 +11,6 @@ import zlib from "zlib";
11
11
 
12
12
  tmp.setGracefulCleanup();
13
13
 
14
- export class NodeStreamSource extends StreamSource {
15
- constructor(stream: Readable, name: string, type?: string, id?: string) {
16
- super(readableToWebStream(stream), name, type, id);
17
- }
18
- }
19
14
 
20
15
  export async function publishMemoryPack(client: VertesiaClient, file: string, name: string): Promise<void> {
21
16
  const stream = createReadStream(file);
@@ -1,32 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.createDocumentTypeFromInteractionRun = createDocumentTypeFromInteractionRun;
4
- const activity_1 = require("@temporalio/activity");
5
- const projections_js_1 = require("../../dsl/projections.js");
6
- const ActivityContext_js_1 = require("../../dsl/setup/ActivityContext.js");
7
- const errors_js_1 = require("../../errors.js");
8
- async function createDocumentTypeFromInteractionRun(payload) {
9
- const { params, client } = await (0, ActivityContext_js_1.setupActivity)(payload);
10
- if (!params.run) {
11
- throw new errors_js_1.ActivityParamNotFound("run", payload.activity);
12
- }
13
- const genTypeRes = params.run.result;
14
- if (!genTypeRes.document_type) {
15
- activity_1.log.error("No name generated for type: " + JSON.stringify(genTypeRes), genTypeRes);
16
- throw new Error("No name generated for type");
17
- }
18
- activity_1.log.info("Generated schema for type", genTypeRes.metadata_schema);
19
- const typeData = {
20
- name: genTypeRes.document_type,
21
- object_schema: genTypeRes.metadata_schema,
22
- is_chunkable: !!genTypeRes.is_chunkable,
23
- };
24
- const type = await client.types.create(typeData);
25
- if (params.updateObjectId) {
26
- await client.objects.update(params.updateObjectId, {
27
- type: type.id,
28
- });
29
- }
30
- return (0, projections_js_1.projectResult)(payload, params, type, { id: type.id, name: type.name });
31
- }
32
- //# sourceMappingURL=createDocumentTypeFromInteractionRun.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"createDocumentTypeFromInteractionRun.js","sourceRoot":"","sources":["../../../../src/activities/advanced/createDocumentTypeFromInteractionRun.ts"],"names":[],"mappings":";;AAuBA,oFA8BC;AApDD,mDAA2C;AAC3C,6DAAyD;AACzD,2EAAmE;AACnE,+CAAwD;AAmBjD,KAAK,UAAU,oCAAoC,CAAC,OAAgF;IACvI,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,IAAA,kCAAa,EAA6C,OAAO,CAAC,CAAC;IAEpG,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;QACd,MAAM,IAAI,iCAAqB,CAAC,KAAK,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC7D,CAAC;IAED,MAAM,UAAU,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC;IAErC,IAAI,CAAC,UAAU,CAAC,aAAa,EAAE,CAAC;QAC5B,cAAG,CAAC,KAAK,CAAC,8BAA8B,GAAG,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC,CAAC;QACnF,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;IAClD,CAAC;IAED,cAAG,CAAC,IAAI,CAAC,2BAA2B,EAAE,UAAU,CAAC,eAAe,CAAC,CAAC;IAClE,MAAM,QAAQ,GAAmC;QAC7C,IAAI,EAAE,UAAU,CAAC,aAAa;QAC9B,aAAa,EAAE,UAAU,CAAC,eAAe;QACzC,YAAY,EAAE,CAAC,CAAC,UAAU,CAAC,YAAY;KAC1C,CAAA;IAED,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAEjD,IAAI,MAAM,CAAC,cAAc,EAAE,CAAC;QACxB,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,cAAc,EAAE;YAC/C,IAAI,EAAE,IAAI,CAAC,EAAE;SAChB,CAAC,CAAC;IACP,CAAC;IAED,OAAO,IAAA,8BAAa,EAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;AAClF,CAAC"}
@@ -1,66 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.createOrUpdateDocumentFromInteractionRun = createOrUpdateDocumentFromInteractionRun;
4
- const common_1 = require("@vertesia/common");
5
- const activity_1 = require("@temporalio/activity");
6
- const ActivityContext_js_1 = require("../../dsl/setup/ActivityContext.js");
7
- const errors_js_1 = require("../../errors.js");
8
- async function createOrUpdateDocumentFromInteractionRun(payload) {
9
- const { params, client } = await (0, ActivityContext_js_1.setupActivity)(payload);
10
- const runId = params.run_id;
11
- const objectTypeName = params.object_type;
12
- if (!runId) {
13
- throw new errors_js_1.ActivityParamNotFound("run_id", payload.activity);
14
- }
15
- if (!objectTypeName && !params.update_existing_id) {
16
- throw new errors_js_1.ActivityParamNotFound("object_type", payload.activity);
17
- }
18
- activity_1.log.info("Creating document from interaction result", { runId, objectTypeName });
19
- const run = await client.runs.retrieve(runId).catch((e) => {
20
- throw new errors_js_1.NoDocumentFound(`Error fetching run ${runId}: ${e.message}`);
21
- });
22
- const type = objectTypeName ?
23
- await client.types.getTypeByName(objectTypeName).catch((e) => {
24
- throw new errors_js_1.NoDocumentFound(`Error fetching type ${objectTypeName}: ${e.message}`);
25
- })
26
- : undefined;
27
- const result = run.result;
28
- const resultIsObject = typeof result === 'object';
29
- const inputData = run.parameters;
30
- let name;
31
- if (resultIsObject) {
32
- name = result['name'] || result["title"] || inputData['name'] || params.fallback_name || 'Untitled';
33
- }
34
- else {
35
- name = inputData['name'] || params.fallback_name || 'Untitled';
36
- }
37
- const docPayload = {
38
- name,
39
- parent: params.parent ?? undefined,
40
- properties: resultIsObject ? result : {},
41
- text: !resultIsObject ? result : undefined,
42
- type: type?.id,
43
- status: common_1.ContentObjectStatus.completed,
44
- generation_run_info: {
45
- id: run.id,
46
- date: new Date().toISOString(),
47
- model: run.modelId,
48
- target: resultIsObject ? 'properties' : 'text'
49
- }
50
- };
51
- //create or update the document
52
- let newDoc = false;
53
- let doc = undefined;
54
- if (params.update_existing_id) {
55
- activity_1.log.info(`Updating existing document ${params.update_existing_id}`);
56
- doc = await client.objects.update(params.update_existing_id, docPayload);
57
- }
58
- else {
59
- activity_1.log.info(`Creating new document of type ${objectTypeName}`);
60
- doc = await client.objects.create(docPayload);
61
- newDoc = true;
62
- }
63
- activity_1.log.info(`Document ${objectTypeName + ' '}${doc.id}(${doc.name}) ${newDoc ? 'created' : 'updated'}`);
64
- return { id: doc.id, isNew: newDoc, type: name };
65
- }
66
- //# sourceMappingURL=createOrUpdateDocumentFromInteractionRun.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"createOrUpdateDocumentFromInteractionRun.js","sourceRoot":"","sources":["../../../../src/activities/advanced/createOrUpdateDocumentFromInteractionRun.ts"],"names":[],"mappings":";;AA8BA,4FAkEC;AAhGD,6CAAqG;AACrG,mDAA2C;AAC3C,2EAAmE;AACnE,+CAAyE;AA2BlE,KAAK,UAAU,wCAAwC,CAAC,OAAkF;IAC7I,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,IAAA,kCAAa,EAA+C,OAAO,CAAC,CAAC;IAEtG,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC;IAC5B,MAAM,cAAc,GAAG,MAAM,CAAC,WAAW,CAAC;IAE1C,IAAI,CAAC,KAAK,EAAE,CAAC;QACT,MAAM,IAAI,iCAAqB,CAAC,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IAChE,CAAC;IACD,IAAI,CAAC,cAAc,IAAI,CAAC,MAAM,CAAC,kBAAkB,EAAE,CAAC;QAChD,MAAM,IAAI,iCAAqB,CAAC,aAAa,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IACrE,CAAC;IAED,cAAG,CAAC,IAAI,CAAC,2CAA2C,EAAE,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,CAAC;IAEjF,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;QACtD,MAAM,IAAI,2BAAe,CAAC,sBAAsB,KAAK,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;IAC3E,CAAC,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,cAAc,CAAC,CAAC;QACzB,MAAM,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,cAAc,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;YACzD,MAAM,IAAI,2BAAe,CAAC,uBAAuB,cAAc,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;QACrF,CAAC,CAAC;QACF,CAAC,CAAC,SAAS,CAAC;IAGhB,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC;IAC1B,MAAM,cAAc,GAAG,OAAO,MAAM,KAAK,QAAQ,CAAC;IAClD,MAAM,SAAS,GAAG,GAAG,CAAC,UAAU,CAAC;IAEjC,IAAI,IAAY,CAAC;IACjB,IAAI,cAAc,EAAE,CAAC;QACjB,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,SAAS,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,aAAa,IAAI,UAAU,CAAC;IACxG,CAAC;SAAM,CAAC;QACJ,IAAI,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,aAAa,IAAI,UAAU,CAAC;IACnE,CAAC;IAED,MAAM,UAAU,GAAG;QACf,IAAI;QACJ,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,SAAS;QAClC,UAAU,EAAE,cAAc,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE;QACxC,IAAI,EAAE,CAAC,cAAc,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;QAC1C,IAAI,EAAE,IAAI,EAAE,EAAE;QACd,MAAM,EAAE,4BAAmB,CAAC,SAAS;QACrC,mBAAmB,EAAE;YACjB,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YAC9B,KAAK,EAAE,GAAG,CAAC,OAAO;YAClB,MAAM,EAAE,cAAc,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM;SACjD;KACJ,CAAC;IAEF,+BAA+B;IAC/B,IAAI,MAAM,GAAY,KAAK,CAAC;IAC5B,IAAI,GAAG,GAAG,SAAS,CAAC;IACpB,IAAI,MAAM,CAAC,kBAAkB,EAAE,CAAC;QAC5B,cAAG,CAAC,IAAI,CAAC,8BAA8B,MAAM,CAAC,kBAAkB,EAAE,CAAC,CAAC;QACpE,GAAG,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,kBAAkB,EAAE,UAAU,CAAC,CAAC;IAC7E,CAAC;SAAM,CAAC;QACJ,cAAG,CAAC,IAAI,CAAC,iCAAiC,cAAc,EAAE,CAAC,CAAC;QAC5D,GAAG,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAC9C,MAAM,GAAG,IAAI,CAAC;IAClB,CAAC;IAED,cAAG,CAAC,IAAI,CAAC,YAAY,cAAc,GAAG,GAAG,GAAG,GAAG,CAAC,EAAE,IAAI,GAAG,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC;IACrG,OAAO,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAA;AACpD,CAAC"}
@@ -1,18 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.updateDocumentFromInteractionRun = updateDocumentFromInteractionRun;
4
- const ActivityContext_js_1 = require("../../dsl/setup/ActivityContext.js");
5
- const errors_js_1 = require("../../errors.js");
6
- async function updateDocumentFromInteractionRun(payload) {
7
- const { params, client, objectId } = await (0, ActivityContext_js_1.setupActivity)(payload);
8
- if (!params.run) {
9
- throw new errors_js_1.ActivityParamNotFound("run", payload.activity);
10
- }
11
- const docProps = params.run.result;
12
- if (!docProps) {
13
- return { status: "failed", error: "no-props" };
14
- }
15
- await client.objects.update(objectId, docProps);
16
- return { status: "success" };
17
- }
18
- //# sourceMappingURL=updateDocumentFromInteractionRun.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"updateDocumentFromInteractionRun.js","sourceRoot":"","sources":["../../../../src/activities/advanced/updateDocumentFromInteractionRun.ts"],"names":[],"mappings":";;AAiBA,4EAgBC;AAhCD,2EAAmE;AACnE,+CAAwD;AAejD,KAAK,UAAU,gCAAgC,CAAC,OAA4E;IAC/H,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAA,kCAAa,EAAyC,OAAO,CAAC,CAAC;IAE1G,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;QACd,MAAM,IAAI,iCAAqB,CAAC,KAAK,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC7D,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC;IAEnC,IAAI,CAAC,QAAQ,EAAE,CAAC;QACZ,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC;IACnD,CAAC;IAED,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAEhD,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;AACjC,CAAC"}
@@ -1,79 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.chunkDocument = chunkDocument;
4
- const activity_1 = require("@temporalio/activity");
5
- const ActivityContext_js_1 = require("../dsl/setup/ActivityContext.js");
6
- const executeInteraction_js_1 = require("./executeInteraction.js");
7
- const INT_CHUNK_DOCUMENT = "sys:ChunkDocument";
8
- async function chunkDocument(payload) {
9
- const { params, client, objectId } = await (0, ActivityContext_js_1.setupActivity)(payload);
10
- const { force } = params;
11
- const interactionName = params.interactionName ?? INT_CHUNK_DOCUMENT;
12
- activity_1.log.info(`Object ${objectId} chunking started`);
13
- const document = await client.objects.retrieve(objectId, "+text");
14
- const type = document.type ? await client.types.retrieve(document.type.id) : undefined;
15
- if (!type?.is_chunkable) {
16
- activity_1.log.warn('Type is not chunkable for object ID: ' + objectId);
17
- return { id: objectId, status: "skipped", message: "type not chunkable" };
18
- }
19
- //check if text is present
20
- if (!document.text) {
21
- activity_1.log.warn('No text found for object ID: ' + objectId);
22
- return { id: objectId, status: "failed", message: "no text found" };
23
- }
24
- if (!force && document.parts && document.parts.length > 0 && document.parts_etag === document.text_etag) {
25
- activity_1.log.info('Document already chunked for object ID: ' + objectId);
26
- return { id: objectId, status: "skipped", message: "document already chunked with correct etag" };
27
- }
28
- //instrument the text with line numbers
29
- const lines = document.text.split('\n');
30
- const instrumented = lines.map((l, i) => `{%${i}%}${l}`).join('\n');
31
- const res = await (0, executeInteraction_js_1.executeInteractionFromActivity)(client, interactionName, params, {
32
- objectId: objectId,
33
- content: instrumented
34
- });
35
- const parts = res.result.parts;
36
- if (!parts || parts.length === 0) {
37
- activity_1.log.warn('No parts found for object ID: ' + objectId, res);
38
- return { id: objectId, status: "failed", parts: [], message: "no parts found" };
39
- }
40
- const partDocs = await Promise.all(parts.map(async (part, i) => {
41
- const text = lines.filter((_l, i) => i >= part.line_number_start && i <= part.line_number_end).join('\n');
42
- const location = () => {
43
- let location = document.location;
44
- if (location.endsWith('/')) {
45
- location += document.name + "/" + part.type;
46
- }
47
- location += '/' + document.name + "/" + part.type;
48
- return location;
49
- };
50
- const docPart = await client.objects.create({
51
- name: part.name,
52
- parent: objectId,
53
- text: text,
54
- location: location(),
55
- properties: {
56
- part_number: i + 1,
57
- etag: document.text_etag,
58
- source_line_start: part.line_number_start,
59
- source_line_end: part.line_number_end,
60
- title: part.name
61
- }
62
- });
63
- return docPart;
64
- }));
65
- //delete previous parts
66
- if (document.parts && document.parts.length > 0) {
67
- activity_1.log.info('Deleting previous parts for object ID: ' + objectId, { parts: document.parts });
68
- await Promise.all(document.parts.map(async (partId) => {
69
- await client.objects.delete(partId);
70
- }));
71
- }
72
- await client.objects.update(objectId, {
73
- parts: partDocs.map(p => p.id),
74
- parts_etag: document.text_etag
75
- });
76
- activity_1.log.info(`Object ${objectId} chunking completed`, { parts: document.parts });
77
- return { id: objectId, status: "completed", parts: document.parts };
78
- }
79
- //# sourceMappingURL=chunkDocument.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"chunkDocument.js","sourceRoot":"","sources":["../../../src/activities/chunkDocument.ts"],"names":[],"mappings":";;AAkCA,sCAyFC;AA1HD,mDAA2C;AAC3C,wEAAgE;AAChE,mEAAqG;AAErG,MAAM,kBAAkB,GAAG,mBAAmB,CAAA;AA6BvC,KAAK,UAAU,aAAa,CAAC,OAAyD;IACzF,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAA,kCAAa,EAAsB,OAAO,CAAC,CAAC;IAEvF,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC;IACzB,MAAM,eAAe,GAAG,MAAM,CAAC,eAAe,IAAI,kBAAkB,CAAC;IAErE,cAAG,CAAC,IAAI,CAAC,UAAU,QAAQ,mBAAmB,CAAC,CAAC;IAEhD,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAElE,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAEvF,IAAI,CAAC,IAAI,EAAE,YAAY,EAAE,CAAC;QACtB,cAAG,CAAC,IAAI,CAAC,uCAAuC,GAAG,QAAQ,CAAC,CAAC;QAC7D,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,oBAAoB,EAAE,CAAA;IAC7E,CAAC;IAED,0BAA0B;IAC1B,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QACjB,cAAG,CAAC,IAAI,CAAC,+BAA+B,GAAG,QAAQ,CAAC,CAAC;QACrD,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,eAAe,EAAE,CAAA;IACvE,CAAC;IAED,IAAI,CAAC,KAAK,IAAI,QAAQ,CAAC,KAAK,IAAI,QAAQ,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,QAAQ,CAAC,UAAU,KAAK,QAAQ,CAAC,SAAS,EAAE,CAAC;QACtG,cAAG,CAAC,IAAI,CAAC,0CAA0C,GAAG,QAAQ,CAAC,CAAC;QAChE,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,4CAA4C,EAAE,CAAA;IACrG,CAAC;IAED,uCAAuC;IACvC,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IACvC,MAAM,YAAY,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAEnE,MAAM,GAAG,GAAG,MAAM,IAAA,sDAA8B,EAAC,MAAM,EAAE,eAAe,EAAE,MAAM,EAAE;QAC9E,QAAQ,EAAE,QAAQ;QAClB,OAAO,EAAE,YAAY;KACxB,CAAC,CAAC;IAEH,MAAM,KAAK,GAAG,GAAG,CAAC,MAAM,CAAC,KAAkB,CAAC;IAC5C,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/B,cAAG,CAAC,IAAI,CAAC,gCAAgC,GAAG,QAAQ,EAAE,GAAG,CAAC,CAAC;QAC3D,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,EAAE,EAAE,OAAO,EAAE,gBAAgB,EAAE,CAAA;IACnF,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE;QAE3D,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,IAAI,CAAC,iBAAiB,IAAI,CAAC,IAAI,IAAI,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE1G,MAAM,QAAQ,GAAG,GAAG,EAAE;YAClB,IAAI,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC;YACjC,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gBACzB,QAAQ,IAAI,QAAQ,CAAC,IAAI,GAAG,GAAG,GAAG,IAAI,CAAC,IAAI,CAAA;YAC/C,CAAC;YACD,QAAQ,IAAI,GAAG,GAAG,QAAQ,CAAC,IAAI,GAAG,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC;YAClD,OAAO,QAAQ,CAAC;QACpB,CAAC,CAAA;QAED,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC;YACxC,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,MAAM,EAAE,QAAQ;YAChB,IAAI,EAAE,IAAI;YACV,QAAQ,EAAE,QAAQ,EAAE;YACpB,UAAU,EAAE;gBACR,WAAW,EAAE,CAAC,GAAG,CAAC;gBAClB,IAAI,EAAE,QAAQ,CAAC,SAAS;gBACxB,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;gBACzC,eAAe,EAAE,IAAI,CAAC,eAAe;gBACrC,KAAK,EAAE,IAAI,CAAC,IAAI;aACnB;SACJ,CAAC,CAAC;QACH,OAAO,OAAO,CAAC;IACnB,CAAC,CAAC,CAAC,CAAC;IAEJ,uBAAuB;IACvB,IAAI,QAAQ,CAAC,KAAK,IAAI,QAAQ,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9C,cAAG,CAAC,IAAI,CAAC,yCAAyC,GAAG,QAAQ,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC;QAC1F,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;YAClD,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACxC,CAAC,CAAC,CAAC,CAAC;IACR,CAAC;IAED,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE;QAClC,KAAK,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9B,UAAU,EAAE,QAAQ,CAAC,SAAS;KACjC,CAAC,CAAC;IAEH,cAAG,CAAC,IAAI,CAAC,UAAU,QAAQ,qBAAqB,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC;IAE7E,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,CAAA;AAEvE,CAAC"}
@@ -1,64 +0,0 @@
1
- "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
4
- };
5
- Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.createPdfDocumentFromSource = createPdfDocumentFromSource;
7
- const activity_1 = require("@temporalio/activity");
8
- const fs_1 = __importDefault(require("fs"));
9
- const mutool_js_1 = require("../conversion/mutool.js");
10
- const ActivityContext_js_1 = require("../dsl/setup/ActivityContext.js");
11
- const errors_js_1 = require("../errors.js");
12
- const blobs_js_1 = require("../utils/blobs.js");
13
- const memory_js_1 = require("../utils/memory.js");
14
- /**
15
- * Create a new PDF by extrracting pages from a source PDF
16
- * @returns
17
- */
18
- async function createPdfDocumentFromSource(payload) {
19
- const { client, objectId, params } = await (0, ActivityContext_js_1.setupActivity)(payload);
20
- const inputObject = await client.objects.retrieve(objectId);
21
- const { pages, filename, title } = params;
22
- activity_1.log.info(`Creating PDF from source`, { objectId, pages, filename, title });
23
- if (!pages || pages.length === 0) {
24
- activity_1.log.error(`No pages provided`);
25
- throw new Error(`No pages provided`);
26
- }
27
- if (!inputObject) {
28
- activity_1.log.error(`Document ${objectId} not found`);
29
- throw new errors_js_1.NoDocumentFound(`Document ${objectId} not found`, [objectId]);
30
- }
31
- if (!inputObject.content?.source) {
32
- activity_1.log.error(`Document ${objectId} has no source`);
33
- throw new errors_js_1.NoDocumentFound(`Document ${objectId} has no source`, [objectId]);
34
- }
35
- if (!inputObject.content.type || (!inputObject.content.type?.startsWith('application/pdf'))) {
36
- activity_1.log.error(`Document ${objectId} is not an image`);
37
- throw new errors_js_1.NoDocumentFound(`Document ${objectId} is not an image or pdf: ${inputObject.content.type}`, [objectId]);
38
- }
39
- const targetType = await client.types.getTypeByName(params.target_object_type);
40
- if (!targetType) {
41
- activity_1.log.error(`Type ${params.target_object_type} not found`);
42
- throw new errors_js_1.NoDocumentFound(`Type ${params.target_object_type} not found`);
43
- }
44
- const tmpFile = await (0, blobs_js_1.saveBlobToTempFile)(client, inputObject.content.source, ".pdf");
45
- const newPdf = await (0, mutool_js_1.pdfExtractPages)(tmpFile, pages);
46
- activity_1.log.info(`PDF created from pages ${pages.join(', ')} `, { newPdf });
47
- const name = `pages-${pages.join('-')}.pdf`;
48
- const sourceToUpload = new memory_js_1.NodeStreamSource(fs_1.default.createReadStream(newPdf), name, "application/pdf");
49
- activity_1.log.info(`Uploading file ${newPdf} `);
50
- const upload = await client.objects.upload(sourceToUpload);
51
- activity_1.log.info(`File uploaded ${upload.source} `);
52
- const newObject = await client.objects.create({
53
- type: targetType.id,
54
- name: title || targetType.name,
55
- parent: objectId,
56
- content: {
57
- source: upload.source,
58
- name: upload.name,
59
- type: 'application/pdf'
60
- }
61
- });
62
- return { newObjectId: newObject.id, uploadedFile: upload.name };
63
- }
64
- //# sourceMappingURL=createDocumentFromOther.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"createDocumentFromOther.js","sourceRoot":"","sources":["../../../src/activities/createDocumentFromOther.ts"],"names":[],"mappings":";;;;;AA6BA,kEA8DC;AA3FD,mDAA2C;AAE3C,4CAAoB;AACpB,uDAA0D;AAC1D,wEAAgE;AAChE,4CAA+C;AAC/C,gDAAuD;AACvD,kDAAsD;AAkBtD;;;GAGG;AACI,KAAK,UAAU,2BAA2B,CAAC,OAAuE;IACrH,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,IAAA,kCAAa,EAAoC,OAAO,CAAC,CAAC;IACrG,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAE5D,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC;IAC1C,cAAG,CAAC,IAAI,CAAC,0BAA0B,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAE3E,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/B,cAAG,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC;QAC/B,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC;IACzC,CAAC;IAED,IAAI,CAAC,WAAW,EAAE,CAAC;QACf,cAAG,CAAC,KAAK,CAAC,YAAY,QAAQ,YAAY,CAAC,CAAC;QAC5C,MAAM,IAAI,2BAAe,CAAC,YAAY,QAAQ,YAAY,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC5E,CAAC;IAED,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;QAC/B,cAAG,CAAC,KAAK,CAAC,YAAY,QAAQ,gBAAgB,CAAC,CAAC;QAChD,MAAM,IAAI,2BAAe,CAAC,YAAY,QAAQ,gBAAgB,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC;IAChF,CAAC;IAED,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC,WAAW,CAAC,OAAO,CAAC,IAAI,EAAE,UAAU,CAAC,iBAAiB,CAAC,CAAC,EAAE,CAAC;QAC1F,cAAG,CAAC,KAAK,CAAC,YAAY,QAAQ,kBAAkB,CAAC,CAAC;QAClD,MAAM,IAAI,2BAAe,CAAC,YAAY,QAAQ,4BAA4B,WAAW,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC;IACtH,CAAC;IAED,MAAM,UAAU,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC;IAC/E,IAAI,CAAC,UAAU,EAAE,CAAC;QACd,cAAG,CAAC,KAAK,CAAC,QAAQ,MAAM,CAAC,kBAAkB,YAAY,CAAC,CAAC;QACzD,MAAM,IAAI,2BAAe,CAAC,QAAQ,MAAM,CAAC,kBAAkB,YAAY,CAAC,CAAC;IAC7E,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,IAAA,6BAAkB,EAAC,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACrF,MAAM,MAAM,GAAG,MAAM,IAAA,2BAAe,EAAC,OAAO,EAAE,KAAK,CAAC,CAAC;IACrD,cAAG,CAAC,IAAI,CAAC,0BAA0B,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;IACpE,MAAM,IAAI,GAAG,SAAS,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC;IAE5C,MAAM,cAAc,GAAG,IAAI,4BAAgB,CACvC,YAAE,CAAC,gBAAgB,CAAC,MAAM,CAAC,EAC3B,IAAI,EACJ,iBAAiB,CACpB,CAAA;IAED,cAAG,CAAC,IAAI,CAAC,kBAAkB,MAAM,GAAG,CAAC,CAAC;IACtC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;IAC3D,cAAG,CAAC,IAAI,CAAC,iBAAiB,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;IAE5C,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC;QAC1C,IAAI,EAAE,UAAU,CAAC,EAAE;QACnB,IAAI,EAAE,KAAK,IAAI,UAAU,CAAC,IAAI;QAC9B,MAAM,EAAE,QAAQ;QAChB,OAAO,EAAE;YACL,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,IAAI,EAAE,iBAAiB;SAC1B;KACJ,CAAC,CAAC;IAEH,OAAO,EAAE,WAAW,EAAE,SAAS,CAAC,EAAE,EAAE,YAAY,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC;AAGpE,CAAC"}
@@ -1,134 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.executeInteraction = executeInteraction;
4
- exports.executeInteractionFromActivity = executeInteractionFromActivity;
5
- const common_1 = require("@vertesia/common");
6
- const activity_1 = require("@temporalio/activity");
7
- const projections_js_1 = require("../dsl/projections.js");
8
- const ActivityContext_js_1 = require("../dsl/setup/ActivityContext.js");
9
- const tokens_js_1 = require("../utils/tokens.js");
10
- //Example:
11
- //@ts-ignore
12
- const JSON = {
13
- name: 'executeInteraction',
14
- import: ["defaultModel", "guidlineId", "docTypeId"],
15
- params: {
16
- defaultModel: "${model}",
17
- interactionName: "GenerateSummary",
18
- model: "${defaultModel ?? 'gpt4'}",
19
- environment: "13456",
20
- max_tokens: 100,
21
- temperature: 0.5,
22
- tags: ["test"],
23
- result_schema: "${docType.object_schema}",
24
- prompt_data: {
25
- documents: "${documents}",
26
- guidline: "${guidline.text}"
27
- }
28
- },
29
- fetch: {
30
- documents: {
31
- type: "document",
32
- query: {
33
- id: { $in: "${objectIds}" },
34
- },
35
- select: "+text",
36
- },
37
- guidline: {
38
- type: "document",
39
- limit: 1,
40
- query: {
41
- id: "${guidlineId}",
42
- },
43
- select: "+text",
44
- on_not_found: "throw"
45
- },
46
- docType: {
47
- type: "document_type",
48
- limit: 1,
49
- query: {
50
- id: "${docTypeId}",
51
- },
52
- select: "+object_schema",
53
- }
54
- }
55
- };
56
- async function executeInteraction(payload) {
57
- const { client, params } = await (0, ActivityContext_js_1.setupActivity)(payload);
58
- const { interactionName, prompt_data } = params;
59
- if (params.truncate) {
60
- const truncate = params.truncate;
61
- for (const [key, value] of Object.entries(truncate)) {
62
- prompt_data[key] = (0, tokens_js_1.truncByMaxTokens)(prompt_data[key], value);
63
- }
64
- }
65
- const res = await executeInteractionFromActivity(client, interactionName, params, prompt_data, payload.debug_mode);
66
- return (0, projections_js_1.projectResult)(payload, params, res, {
67
- runId: res.id,
68
- status: res.status,
69
- result: res.result,
70
- });
71
- }
72
- async function executeInteractionFromActivity(client, interactionName, params, prompt_data, debug) {
73
- const userTags = params.tags;
74
- const info = (0, activity_1.activityInfo)();
75
- const runId = info.workflowExecution.runId;
76
- let tags = ["workflow", `tmpRunId:${runId}`]; //TODO use wf:wfName
77
- if (userTags) {
78
- tags = tags.concat(userTags);
79
- }
80
- let previousStudioExecutionRun = undefined;
81
- if (params.include_previous_error) {
82
- //retrieve last failed run if any
83
- if (info.attempt > 1) {
84
- activity_1.log.info("Retrying, searching for previous run", { tags: ["tmpRunId:" + runId] });
85
- const payload = {
86
- query: { tags: ["tmpRunId:" + info.workflowExecution.runId] },
87
- limit: 1,
88
- };
89
- const previousRun = await client.runs.search(payload).then((res) => {
90
- activity_1.log.info("Search results", { results: res });
91
- return res ? res[0] ?? undefined : undefined;
92
- });
93
- if (previousRun) {
94
- activity_1.log.info("Found previous run", { previousRun });
95
- previousStudioExecutionRun = await client.runs.retrieve(previousRun.id);
96
- }
97
- }
98
- }
99
- if (debug && previousStudioExecutionRun?.error) {
100
- activity_1.log.info(`Found previous run error`, { error: previousStudioExecutionRun?.error });
101
- }
102
- const config = {
103
- environment: params.environment,
104
- model: params.model,
105
- model_options: params.model_options,
106
- };
107
- const data = {
108
- ...prompt_data,
109
- previous_error: previousStudioExecutionRun?.error,
110
- };
111
- const result_schema = params.result_schema;
112
- if (debug) {
113
- activity_1.log.info(`About to execute interaction ${interactionName}`, { config, data, result_schema, tags });
114
- }
115
- const res = await client.interactions.executeByName(interactionName, {
116
- config,
117
- data,
118
- result_schema,
119
- tags,
120
- stream: false,
121
- }).catch((err) => {
122
- activity_1.log.error(`Error executing interaction ${interactionName}`, { err });
123
- throw new Error(`Interaction Execution failed ${interactionName}: ${err.message}`);
124
- });
125
- if (debug) {
126
- activity_1.log.info(`Interaction executed ${interactionName}`, res);
127
- }
128
- if (res.error || res.status === common_1.ExecutionRunStatus.failed) {
129
- activity_1.log.error(`Error executing interaction ${interactionName}`, { error: res.error });
130
- throw new Error(`Interaction Execution failed ${interactionName}: ${res.error}`);
131
- }
132
- return res;
133
- }
134
- //# sourceMappingURL=executeInteraction.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"executeInteraction.js","sourceRoot":"","sources":["../../../src/activities/executeInteraction.ts"],"names":[],"mappings":";;AAmGA,gDAsBC;AAED,wEAsEC;AAhMD,6CAAuK;AACvK,mDAAyD;AACzD,0DAAsD;AACtD,wEAAgE;AAChE,kDAAoE;AAGpE,UAAU;AACV,YAAY;AACZ,MAAM,IAAI,GAAoB;IAC1B,IAAI,EAAE,oBAAoB;IAC1B,MAAM,EAAE,CAAC,cAAc,EAAE,YAAY,EAAE,WAAW,CAAC;IACnD,MAAM,EAAE;QACJ,YAAY,EAAE,UAAU;QACxB,eAAe,EAAE,iBAAiB;QAClC,KAAK,EAAE,2BAA2B;QAClC,WAAW,EAAE,OAAO;QACpB,UAAU,EAAE,GAAG;QACf,WAAW,EAAE,GAAG;QAChB,IAAI,EAAE,CAAC,MAAM,CAAC;QACd,aAAa,EAAE,0BAA0B;QACzC,WAAW,EAAE;YACT,SAAS,EAAE,cAAc;YACzB,QAAQ,EAAE,kBAAkB;SAC/B;KACJ;IACD,KAAK,EAAE;QACH,SAAS,EAAE;YACP,IAAI,EAAE,UAAU;YAChB,KAAK,EAAE;gBACH,EAAE,EAAE,EAAE,GAAG,EAAE,cAAc,EAAE;aAC9B;YACD,MAAM,EAAE,OAAO;SAClB;QACD,QAAQ,EAAE;YACN,IAAI,EAAE,UAAU;YAChB,KAAK,EAAE,CAAC;YACR,KAAK,EAAE;gBACH,EAAE,EAAE,eAAe;aACtB;YACD,MAAM,EAAE,OAAO;YACf,YAAY,EAAE,OAAO;SACxB;QACD,OAAO,EAAE;YACL,IAAI,EAAE,eAAe;YACrB,KAAK,EAAE,CAAC;YACR,KAAK,EAAE;gBACH,EAAE,EAAE,cAAc;aACrB;YACD,MAAM,EAAE,gBAAgB;SAC3B;KACJ;CACJ,CAAA;AA8CM,KAAK,UAAU,kBAAkB,CAAC,OAA8D;IACnG,MAAM,EACF,MAAM,EAAE,MAAM,EACjB,GAAG,MAAM,IAAA,kCAAa,EAA2B,OAAO,CAAC,CAAC;IAE3D,MAAM,EAAE,eAAe,EAAE,WAAW,EAAE,GAAG,MAAM,CAAC;IAEhD,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;QAClB,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;QACjC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;YAClD,WAAW,CAAC,GAAG,CAAC,GAAG,IAAA,4BAAgB,EAAC,WAAW,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,CAAC;QACjE,CAAC;IACL,CAAC;IAED,MAAM,GAAG,GAAG,MAAM,8BAA8B,CAAC,MAAM,EAAE,eAAe,EAAE,MAAM,EAAE,WAAW,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;IAEnH,OAAO,IAAA,8BAAa,EAAC,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE;QACvC,KAAK,EAAE,GAAG,CAAC,EAAE;QACb,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,MAAM,EAAE,GAAG,CAAC,MAAM;KACrB,CAAC,CAAC;AAEP,CAAC;AAEM,KAAK,UAAU,8BAA8B,CAAC,MAAsB,EAAE,eAAuB,EAAE,MAAkC,EAAE,WAAgB,EAAE,KAAe;IACvK,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC;IAC7B,MAAM,IAAI,GAAG,IAAA,uBAAY,GAAE,CAAC;IAC5B,MAAM,KAAK,GAAG,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC;IAC3C,IAAI,IAAI,GAAG,CAAC,UAAU,EAAE,YAAY,KAAK,EAAE,CAAC,CAAC,CAAC,oBAAoB;IAClE,IAAI,QAAQ,EAAE,CAAC;QACX,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IACjC,CAAC;IAED,IAAI,0BAA0B,GAA6B,SAAS,CAAC;IACrE,IAAI,MAAM,CAAC,sBAAsB,EAAE,CAAC;QAChC,iCAAiC;QACjC,IAAI,IAAI,CAAC,OAAO,GAAG,CAAC,EAAE,CAAC;YACnB,cAAG,CAAC,IAAI,CAAC,sCAAsC,EAAE,EAAE,IAAI,EAAE,CAAC,WAAW,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC;YAClF,MAAM,OAAO,GAAqB;gBAC9B,KAAK,EAAE,EAAE,IAAI,EAAE,CAAC,WAAW,GAAG,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,EAAE;gBAC7D,KAAK,EAAE,CAAC;aACX,CAAC;YACF,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE;gBAC/D,cAAG,CAAC,IAAI,CAAC,gBAAgB,EAAE,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC;gBAC7C,OAAO,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,SAAS,CAAC,CAAC,CAAC,SAAS,CAAA;YAChD,CAAC,CAAC,CAAC;YAEH,IAAI,WAAW,EAAE,CAAC;gBACd,cAAG,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC;gBAChD,0BAA0B,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;YAC5E,CAAC;QACL,CAAC;IACL,CAAC;IACD,IAAI,KAAK,IAAI,0BAA0B,EAAE,KAAK,EAAE,CAAC;QAC7C,cAAG,CAAC,IAAI,CAAC,2BAA2B,EAAE,EAAE,KAAK,EAAE,0BAA0B,EAAE,KAAK,EAAE,CAAC,CAAC;IACxF,CAAC;IAED,MAAM,MAAM,GAAsC;QAC9C,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,aAAa,EAAE,MAAM,CAAC,aAAa;KACtC,CAAA;IACD,MAAM,IAAI,GAAG;QACT,GAAG,WAAW;QACd,cAAc,EAAE,0BAA0B,EAAE,KAAK;KACpD,CAAA;IAED,MAAM,aAAa,GAAG,MAAM,CAAC,aAAa,CAAC;IAE3C,IAAI,KAAK,EAAE,CAAC;QACR,cAAG,CAAC,IAAI,CAAC,gCAAgC,eAAe,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IACvG,CAAC;IAED,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,YAAY,CAAC,aAAa,CAAC,eAAe,EAAE;QACjE,MAAM;QACN,IAAI;QACJ,aAAa;QACb,IAAI;QACJ,MAAM,EAAE,KAAK;KAChB,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;QACb,cAAG,CAAC,KAAK,CAAC,+BAA+B,eAAe,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;QACrE,MAAM,IAAI,KAAK,CAAC,gCAAgC,eAAe,KAAK,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;IACvF,CAAC,CAAC,CAAC;IAEH,IAAI,KAAK,EAAE,CAAC;QACR,cAAG,CAAC,IAAI,CAAC,wBAAwB,eAAe,EAAE,EAAE,GAAG,CAAC,CAAC;IAC7D,CAAC;IAED,IAAI,GAAG,CAAC,KAAK,IAAI,GAAG,CAAC,MAAM,KAAK,2BAAkB,CAAC,MAAM,EAAE,CAAC;QACxD,cAAG,CAAC,KAAK,CAAC,+BAA+B,eAAe,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;QAClF,MAAM,IAAI,KAAK,CAAC,gCAAgC,eAAe,KAAK,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;IACrF,CAAC;IAED,OAAO,GAAG,CAAC;AACf,CAAC"}
@@ -1,135 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.extractDocumentText = extractDocumentText;
4
- const activity_1 = require("@temporalio/activity");
5
- const mutool_js_1 = require("../conversion/mutool.js");
6
- const pandoc_js_1 = require("../conversion/pandoc.js");
7
- const ActivityContext_js_1 = require("../dsl/setup/ActivityContext.js");
8
- const errors_js_1 = require("../errors.js");
9
- const result_types_js_1 = require("../result-types.js");
10
- const blobs_js_1 = require("../utils/blobs.js");
11
- const tokens_js_1 = require("../utils/tokens.js");
12
- //@ts-ignore
13
- const JSON = {
14
- name: 'extractDocumentText',
15
- };
16
- ;
17
- async function extractDocumentText(payload) {
18
- const { client, objectId } = await (0, ActivityContext_js_1.setupActivity)(payload);
19
- const r = await client.objects.find({
20
- query: { _id: objectId },
21
- limit: 1,
22
- select: "+text"
23
- });
24
- const doc = r[0];
25
- if (!doc) {
26
- activity_1.log.error(`Document ${objectId} not found`);
27
- throw new errors_js_1.NoDocumentFound(`Document ${objectId} not found`, payload.objectIds);
28
- }
29
- activity_1.log.info(`Extracting text for object ${doc.id}`);
30
- if (!doc.content?.type || !doc.content?.source) {
31
- if (doc.text) {
32
- return createResponse(doc, doc.text, result_types_js_1.TextExtractionStatus.skipped, "Text present and no source or type");
33
- }
34
- else {
35
- return createResponse(doc, "", result_types_js_1.TextExtractionStatus.error, "No source or type found");
36
- }
37
- }
38
- //skip if text already extracted and proper etag
39
- if (doc.text && doc.text.length > 0 && doc.text_etag === doc.content.etag) {
40
- return createResponse(doc, doc.text, result_types_js_1.TextExtractionStatus.skipped, "Text already extracted");
41
- }
42
- let fileBuffer;
43
- try {
44
- fileBuffer = await (0, blobs_js_1.fetchBlobAsBuffer)(client, doc.content.source);
45
- }
46
- catch (e) {
47
- activity_1.log.error(`Error reading file: ${e}`);
48
- return createResponse(doc, "", result_types_js_1.TextExtractionStatus.error, e.message);
49
- }
50
- let txt;
51
- switch (doc.content.type) {
52
- case 'application/pdf':
53
- //if pdf is more than 2MB, use mutool
54
- txt = await (0, mutool_js_1.mutoolPdfToText)(fileBuffer);
55
- break;
56
- case 'text/plain':
57
- txt = fileBuffer.toString('utf8');
58
- break;
59
- //docx
60
- case 'application/vnd.openxmlformats-officedocument.wordprocessingml.document':
61
- txt = await (0, pandoc_js_1.manyToMarkdown)(fileBuffer, 'docx');
62
- break;
63
- //html
64
- case 'text/html':
65
- txt = await (0, pandoc_js_1.manyToMarkdown)(fileBuffer, 'html');
66
- break;
67
- //opendocument
68
- case 'application/vnd.oasis.opendocument.text':
69
- txt = await (0, pandoc_js_1.manyToMarkdown)(fileBuffer, 'odt');
70
- break;
71
- //rtf
72
- case 'application/rtf':
73
- txt = await (0, pandoc_js_1.manyToMarkdown)(fileBuffer, 'rtf');
74
- break;
75
- //markdown
76
- case 'text/markdown':
77
- txt = fileBuffer.toString('utf8');
78
- break;
79
- //csv
80
- case 'text/csv':
81
- txt = fileBuffer.toString('utf8');
82
- break;
83
- //typescript
84
- case 'application/typescript':
85
- txt = fileBuffer.toString('utf8');
86
- break;
87
- //javascript
88
- case 'application/javascript':
89
- txt = fileBuffer.toString('utf8');
90
- break;
91
- //json
92
- case 'application/json':
93
- txt = fileBuffer.toString('utf8');
94
- break;
95
- default:
96
- if (sniffIfText(fileBuffer)) {
97
- txt = fileBuffer.toString('utf8'); //TODO: add charset detection
98
- break;
99
- }
100
- return createResponse(doc, doc.text ?? '', result_types_js_1.TextExtractionStatus.skipped, `Unsupported mime type: ${doc.content.type}`);
101
- }
102
- const tokensData = (0, tokens_js_1.countTokens)(txt);
103
- const etag = doc.content.etag ?? (0, blobs_js_1.md5)(txt);
104
- const updateData = {
105
- text: txt,
106
- text_etag: etag,
107
- tokens: {
108
- ...tokensData,
109
- etag: etag,
110
- }
111
- };
112
- await client.objects.update(doc.id, updateData);
113
- return createResponse(doc, txt, result_types_js_1.TextExtractionStatus.success);
114
- }
115
- function createResponse(doc, text, status, message) {
116
- return {
117
- status,
118
- message,
119
- tokens: doc.tokens,
120
- len: text.length,
121
- objectId: doc.id,
122
- hasText: !!text,
123
- };
124
- }
125
- //if file is less than 100KB, check if it looks like text
126
- function sniffIfText(buf) {
127
- if (buf.length < 100 * 1024) {
128
- const s = buf.toString('utf8');
129
- if (s.length > 0) {
130
- return true;
131
- }
132
- }
133
- return false;
134
- }
135
- //# sourceMappingURL=extractDocumentText.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"extractDocumentText.js","sourceRoot":"","sources":["../../../src/activities/extractDocumentText.ts"],"names":[],"mappings":";;AAsBA,kDAyHC;AA/ID,mDAA2C;AAE3C,uDAA0D;AAC1D,uDAAyD;AACzD,wEAAgE;AAChE,4CAA+C;AAC/C,wDAAgF;AAChF,gDAA2D;AAC3D,kDAAiD;AAEjD,YAAY;AACZ,MAAM,IAAI,GAAoB;IAC1B,IAAI,EAAE,qBAAqB;CAC9B,CAAA;AAG6C,CAAC;AAMxC,KAAK,UAAU,mBAAmB,CAAC,OAA+D;IACrG,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAA,kCAAa,EAAC,OAAO,CAAC,CAAC;IAE1D,MAAM,CAAC,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC;QAChC,KAAK,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE;QACxB,KAAK,EAAE,CAAC;QACR,MAAM,EAAE,OAAO;KAClB,CAAC,CAAA;IACF,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,CAAkB,CAAC;IAClC,IAAI,CAAC,GAAG,EAAE,CAAC;QACP,cAAG,CAAC,KAAK,CAAC,YAAY,QAAQ,YAAY,CAAC,CAAC;QAC5C,MAAM,IAAI,2BAAe,CAAC,YAAY,QAAQ,YAAY,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;IACnF,CAAC;IAED,cAAG,CAAC,IAAI,CAAC,8BAA8B,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC;IAGjD,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;QAC7C,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;YACX,OAAO,cAAc,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE,sCAAoB,CAAC,OAAO,EAAE,oCAAoC,CAAC,CAAC;QAC7G,CAAC;aAAM,CAAC;YACJ,OAAO,cAAc,CAAC,GAAG,EAAE,EAAE,EAAE,sCAAoB,CAAC,KAAK,EAAE,yBAAyB,CAAC,CAAC;QAC1F,CAAC;IACL,CAAC;IAED,gDAAgD;IAChD,IAAI,GAAG,CAAC,IAAI,IAAI,GAAG,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,GAAG,CAAC,SAAS,KAAK,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QACxE,OAAO,cAAc,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE,sCAAoB,CAAC,OAAO,EAAE,wBAAwB,CAAC,CAAC;IACjG,CAAC;IAED,IAAI,UAAkB,CAAC;IACvB,IAAI,CAAC;QACD,UAAU,GAAG,MAAM,IAAA,4BAAiB,EAAC,MAAM,EAAE,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACrE,CAAC;IAAC,OAAO,CAAM,EAAE,CAAC;QACd,cAAG,CAAC,KAAK,CAAC,uBAAuB,CAAC,EAAE,CAAC,CAAC;QACtC,OAAO,cAAc,CAAC,GAAG,EAAE,EAAE,EAAE,sCAAoB,CAAC,KAAK,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC;IAC1E,CAAC;IAGD,IAAI,GAAW,CAAC;IAEhB,QAAQ,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAEvB,KAAK,iBAAiB;YAClB,qCAAqC;YACrC,GAAG,GAAG,MAAM,IAAA,2BAAe,EAAC,UAAU,CAAC,CAAC;YACxC,MAAM;QAEV,KAAK,YAAY;YACb,GAAG,GAAG,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAA;YACjC,MAAM;QAEV,MAAM;QACN,KAAK,yEAAyE;YAC1E,GAAG,GAAG,MAAM,IAAA,0BAAc,EAAC,UAAU,EAAE,MAAM,CAAC,CAAC;YAC/C,MAAM;QAEV,MAAM;QACN,KAAK,WAAW;YACZ,GAAG,GAAG,MAAM,IAAA,0BAAc,EAAC,UAAU,EAAE,MAAM,CAAC,CAAC;YAC/C,MAAM;QAEV,cAAc;QACd,KAAK,yCAAyC;YAC1C,GAAG,GAAG,MAAM,IAAA,0BAAc,EAAC,UAAU,EAAE,KAAK,CAAC,CAAC;YAC9C,MAAM;QAEV,KAAK;QACL,KAAK,iBAAiB;YAClB,GAAG,GAAG,MAAM,IAAA,0BAAc,EAAC,UAAU,EAAE,KAAK,CAAC,CAAC;YAC9C,MAAM;QAEV,UAAU;QACV,KAAK,eAAe;YAChB,GAAG,GAAG,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAClC,MAAM;QAEV,KAAK;QACL,KAAK,UAAU;YACX,GAAG,GAAG,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAClC,MAAM;QAEV,YAAY;QACZ,KAAK,wBAAwB;YACzB,GAAG,GAAG,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAClC,MAAM;QAEV,YAAY;QACZ,KAAK,wBAAwB;YACzB,GAAG,GAAG,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAClC,MAAM;QAEV,MAAM;QACN,KAAK,kBAAkB;YACnB,GAAG,GAAG,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAClC,MAAM;QAEV;YACI,IAAI,WAAW,CAAC,UAAU,CAAC,EAAE,CAAC;gBAC1B,GAAG,GAAG,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,6BAA6B;gBAChE,MAAM;YACV,CAAC;YACD,OAAO,cAAc,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,sCAAoB,CAAC,OAAO,EAAE,0BAA0B,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IAC/H,CAAC;IAGD,MAAM,UAAU,GAAG,IAAA,uBAAW,EAAC,GAAG,CAAC,CAAC;IACpC,MAAM,IAAI,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,IAAI,IAAA,cAAG,EAAC,GAAG,CAAC,CAAC;IAE1C,MAAM,UAAU,GAA+B;QAC3C,IAAI,EAAE,GAAG;QACT,SAAS,EAAE,IAAI;QACf,MAAM,EAAE;YACJ,GAAG,UAAU;YACb,IAAI,EAAE,IAAI;SACb;KACJ,CAAA;IAED,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;IAEhD,OAAO,cAAc,CAAC,GAAG,EAAE,GAAG,EAAE,sCAAoB,CAAC,OAAO,CAAC,CAAC;AAClE,CAAC;AAED,SAAS,cAAc,CAAC,GAAkB,EAAE,IAAY,EAAE,MAA4B,EAAE,OAAgB;IACpG,OAAO;QACH,MAAM;QACN,OAAO;QACP,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,GAAG,EAAE,IAAI,CAAC,MAAM;QAChB,QAAQ,EAAE,GAAG,CAAC,EAAE;QAChB,OAAO,EAAE,CAAC,CAAC,IAAI;KAClB,CAAA;AAEL,CAAC;AAGD,yDAAyD;AACzD,SAAS,WAAW,CAAC,GAAW;IAC5B,IAAI,GAAG,CAAC,MAAM,GAAG,GAAG,GAAG,IAAI,EAAE,CAAC;QAC1B,MAAM,CAAC,GAAG,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAC/B,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACf,OAAO,IAAI,CAAC;QAChB,CAAC;IACL,CAAC;IACD,OAAO,KAAK,CAAC;AACjB,CAAC"}