@vertesia/workflow 0.24.0-dev.202601221707

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (490) hide show
  1. package/LICENSE +13 -0
  2. package/README.md +65 -0
  3. package/bin/bundle-workflows.mjs +39 -0
  4. package/lib/cjs/activities/advanced/createDocumentTypeFromInteractionRun.js +33 -0
  5. package/lib/cjs/activities/advanced/createDocumentTypeFromInteractionRun.js.map +1 -0
  6. package/lib/cjs/activities/advanced/createOrUpdateDocumentFromInteractionRun.js +73 -0
  7. package/lib/cjs/activities/advanced/createOrUpdateDocumentFromInteractionRun.js.map +1 -0
  8. package/lib/cjs/activities/advanced/updateDocumentFromInteractionRun.js +19 -0
  9. package/lib/cjs/activities/advanced/updateDocumentFromInteractionRun.js.map +1 -0
  10. package/lib/cjs/activities/chunkDocument.js +85 -0
  11. package/lib/cjs/activities/chunkDocument.js.map +1 -0
  12. package/lib/cjs/activities/copyParentArtifacts.js +127 -0
  13. package/lib/cjs/activities/copyParentArtifacts.js.map +1 -0
  14. package/lib/cjs/activities/createDocumentFromOther.js +64 -0
  15. package/lib/cjs/activities/createDocumentFromOther.js.map +1 -0
  16. package/lib/cjs/activities/executeInteraction.js +194 -0
  17. package/lib/cjs/activities/executeInteraction.js.map +1 -0
  18. package/lib/cjs/activities/extractDocumentText.js +156 -0
  19. package/lib/cjs/activities/extractDocumentText.js.map +1 -0
  20. package/lib/cjs/activities/generateDocumentProperties.js +83 -0
  21. package/lib/cjs/activities/generateDocumentProperties.js.map +1 -0
  22. package/lib/cjs/activities/generateEmbeddings.js +257 -0
  23. package/lib/cjs/activities/generateEmbeddings.js.map +1 -0
  24. package/lib/cjs/activities/generateOrAssignContentType.js +125 -0
  25. package/lib/cjs/activities/generateOrAssignContentType.js.map +1 -0
  26. package/lib/cjs/activities/getObjectFromStore.js +20 -0
  27. package/lib/cjs/activities/getObjectFromStore.js.map +1 -0
  28. package/lib/cjs/activities/handleError.js +22 -0
  29. package/lib/cjs/activities/handleError.js.map +1 -0
  30. package/lib/cjs/activities/index-dsl.js +51 -0
  31. package/lib/cjs/activities/index-dsl.js.map +1 -0
  32. package/lib/cjs/activities/index.js +21 -0
  33. package/lib/cjs/activities/index.js.map +1 -0
  34. package/lib/cjs/activities/media/prepareAudio.js +239 -0
  35. package/lib/cjs/activities/media/prepareAudio.js.map +1 -0
  36. package/lib/cjs/activities/media/prepareVideo.js +429 -0
  37. package/lib/cjs/activities/media/prepareVideo.js.map +1 -0
  38. package/lib/cjs/activities/media/processPdfWithTextract.js +103 -0
  39. package/lib/cjs/activities/media/processPdfWithTextract.js.map +1 -0
  40. package/lib/cjs/activities/media/saveGladiaTranscription.js +81 -0
  41. package/lib/cjs/activities/media/saveGladiaTranscription.js.map +1 -0
  42. package/lib/cjs/activities/media/transcribeMediaWithGladia.js +82 -0
  43. package/lib/cjs/activities/media/transcribeMediaWithGladia.js.map +1 -0
  44. package/lib/cjs/activities/notifyWebhook.js +158 -0
  45. package/lib/cjs/activities/notifyWebhook.js.map +1 -0
  46. package/lib/cjs/activities/rateLimiter.js +30 -0
  47. package/lib/cjs/activities/rateLimiter.js.map +1 -0
  48. package/lib/cjs/activities/renditions/generateImageRendition.js +66 -0
  49. package/lib/cjs/activities/renditions/generateImageRendition.js.map +1 -0
  50. package/lib/cjs/activities/renditions/generateVideoRendition.js +200 -0
  51. package/lib/cjs/activities/renditions/generateVideoRendition.js.map +1 -0
  52. package/lib/cjs/activities/setDocumentStatus.js +15 -0
  53. package/lib/cjs/activities/setDocumentStatus.js.map +1 -0
  54. package/lib/cjs/conversion/TextractProcessor.js +417 -0
  55. package/lib/cjs/conversion/TextractProcessor.js.map +1 -0
  56. package/lib/cjs/conversion/image.js +149 -0
  57. package/lib/cjs/conversion/image.js.map +1 -0
  58. package/lib/cjs/conversion/markitdown.js +42 -0
  59. package/lib/cjs/conversion/markitdown.js.map +1 -0
  60. package/lib/cjs/conversion/mutool.js +147 -0
  61. package/lib/cjs/conversion/mutool.js.map +1 -0
  62. package/lib/cjs/conversion/pandoc.js +39 -0
  63. package/lib/cjs/conversion/pandoc.js.map +1 -0
  64. package/lib/cjs/dsl/conditions.js +81 -0
  65. package/lib/cjs/dsl/conditions.js.map +1 -0
  66. package/lib/cjs/dsl/dsl-workflow.js +343 -0
  67. package/lib/cjs/dsl/dsl-workflow.js.map +1 -0
  68. package/lib/cjs/dsl/dslProxyActivities.js +23 -0
  69. package/lib/cjs/dsl/dslProxyActivities.js.map +1 -0
  70. package/lib/cjs/dsl/projections.js +59 -0
  71. package/lib/cjs/dsl/projections.js.map +1 -0
  72. package/lib/cjs/dsl/setup/ActivityContext.js +122 -0
  73. package/lib/cjs/dsl/setup/ActivityContext.js.map +1 -0
  74. package/lib/cjs/dsl/setup/fetch/DataProvider.js +51 -0
  75. package/lib/cjs/dsl/setup/fetch/DataProvider.js.map +1 -0
  76. package/lib/cjs/dsl/setup/fetch/index.js +16 -0
  77. package/lib/cjs/dsl/setup/fetch/index.js.map +1 -0
  78. package/lib/cjs/dsl/setup/fetch/providers.js +67 -0
  79. package/lib/cjs/dsl/setup/fetch/providers.js.map +1 -0
  80. package/lib/cjs/dsl/test/test-child-workflow.js +10 -0
  81. package/lib/cjs/dsl/test/test-child-workflow.js.map +1 -0
  82. package/lib/cjs/dsl/validation.js +122 -0
  83. package/lib/cjs/dsl/validation.js.map +1 -0
  84. package/lib/cjs/dsl/vars.js +341 -0
  85. package/lib/cjs/dsl/vars.js.map +1 -0
  86. package/lib/cjs/dsl/walk.js +100 -0
  87. package/lib/cjs/dsl/walk.js.map +1 -0
  88. package/lib/cjs/dsl.js +20 -0
  89. package/lib/cjs/dsl.js.map +1 -0
  90. package/lib/cjs/errors.js +79 -0
  91. package/lib/cjs/errors.js.map +1 -0
  92. package/lib/cjs/index.js +56 -0
  93. package/lib/cjs/index.js.map +1 -0
  94. package/lib/cjs/iterative-generation/activities/extractToc.js +47 -0
  95. package/lib/cjs/iterative-generation/activities/extractToc.js.map +1 -0
  96. package/lib/cjs/iterative-generation/activities/finalizeOutput.js +72 -0
  97. package/lib/cjs/iterative-generation/activities/finalizeOutput.js.map +1 -0
  98. package/lib/cjs/iterative-generation/activities/generatePart.js +78 -0
  99. package/lib/cjs/iterative-generation/activities/generatePart.js.map +1 -0
  100. package/lib/cjs/iterative-generation/activities/generateToc.js +86 -0
  101. package/lib/cjs/iterative-generation/activities/generateToc.js.map +1 -0
  102. package/lib/cjs/iterative-generation/activities/index.js +12 -0
  103. package/lib/cjs/iterative-generation/activities/index.js.map +1 -0
  104. package/lib/cjs/iterative-generation/iterativeGenerationWorkflow.js +56 -0
  105. package/lib/cjs/iterative-generation/iterativeGenerationWorkflow.js.map +1 -0
  106. package/lib/cjs/iterative-generation/types.js +5 -0
  107. package/lib/cjs/iterative-generation/types.js.map +1 -0
  108. package/lib/cjs/iterative-generation/utils.js +121 -0
  109. package/lib/cjs/iterative-generation/utils.js.map +1 -0
  110. package/lib/cjs/package.json +3 -0
  111. package/lib/cjs/result-types.js +10 -0
  112. package/lib/cjs/result-types.js.map +1 -0
  113. package/lib/cjs/system/notifyWebhookWorkflow.js +53 -0
  114. package/lib/cjs/system/notifyWebhookWorkflow.js.map +1 -0
  115. package/lib/cjs/system/recalculateEmbeddingsWorkflow.js +33 -0
  116. package/lib/cjs/system/recalculateEmbeddingsWorkflow.js.map +1 -0
  117. package/lib/cjs/utils/auth.js +15 -0
  118. package/lib/cjs/utils/auth.js.map +1 -0
  119. package/lib/cjs/utils/blobs.js +64 -0
  120. package/lib/cjs/utils/blobs.js.map +1 -0
  121. package/lib/cjs/utils/chunks.js +14 -0
  122. package/lib/cjs/utils/chunks.js.map +1 -0
  123. package/lib/cjs/utils/client.js +31 -0
  124. package/lib/cjs/utils/client.js.map +1 -0
  125. package/lib/cjs/utils/expand-vars.js +33 -0
  126. package/lib/cjs/utils/expand-vars.js.map +1 -0
  127. package/lib/cjs/utils/memory.js +65 -0
  128. package/lib/cjs/utils/memory.js.map +1 -0
  129. package/lib/cjs/utils/renditions.js +88 -0
  130. package/lib/cjs/utils/renditions.js.map +1 -0
  131. package/lib/cjs/utils/storage.js +54 -0
  132. package/lib/cjs/utils/storage.js.map +1 -0
  133. package/lib/cjs/utils/tokens.js +38 -0
  134. package/lib/cjs/utils/tokens.js.map +1 -0
  135. package/lib/cjs/vars.js +20 -0
  136. package/lib/cjs/vars.js.map +1 -0
  137. package/lib/cjs/workflows.js +15 -0
  138. package/lib/cjs/workflows.js.map +1 -0
  139. package/lib/esm/activities/advanced/createDocumentTypeFromInteractionRun.js +30 -0
  140. package/lib/esm/activities/advanced/createDocumentTypeFromInteractionRun.js.map +1 -0
  141. package/lib/esm/activities/advanced/createOrUpdateDocumentFromInteractionRun.js +70 -0
  142. package/lib/esm/activities/advanced/createOrUpdateDocumentFromInteractionRun.js.map +1 -0
  143. package/lib/esm/activities/advanced/updateDocumentFromInteractionRun.js +16 -0
  144. package/lib/esm/activities/advanced/updateDocumentFromInteractionRun.js.map +1 -0
  145. package/lib/esm/activities/chunkDocument.js +82 -0
  146. package/lib/esm/activities/chunkDocument.js.map +1 -0
  147. package/lib/esm/activities/copyParentArtifacts.js +124 -0
  148. package/lib/esm/activities/copyParentArtifacts.js.map +1 -0
  149. package/lib/esm/activities/createDocumentFromOther.js +58 -0
  150. package/lib/esm/activities/createDocumentFromOther.js.map +1 -0
  151. package/lib/esm/activities/executeInteraction.js +190 -0
  152. package/lib/esm/activities/executeInteraction.js.map +1 -0
  153. package/lib/esm/activities/extractDocumentText.js +153 -0
  154. package/lib/esm/activities/extractDocumentText.js.map +1 -0
  155. package/lib/esm/activities/generateDocumentProperties.js +80 -0
  156. package/lib/esm/activities/generateDocumentProperties.js.map +1 -0
  157. package/lib/esm/activities/generateEmbeddings.js +254 -0
  158. package/lib/esm/activities/generateEmbeddings.js.map +1 -0
  159. package/lib/esm/activities/generateOrAssignContentType.js +122 -0
  160. package/lib/esm/activities/generateOrAssignContentType.js.map +1 -0
  161. package/lib/esm/activities/getObjectFromStore.js +17 -0
  162. package/lib/esm/activities/getObjectFromStore.js.map +1 -0
  163. package/lib/esm/activities/handleError.js +19 -0
  164. package/lib/esm/activities/handleError.js.map +1 -0
  165. package/lib/esm/activities/index-dsl.js +25 -0
  166. package/lib/esm/activities/index-dsl.js.map +1 -0
  167. package/lib/esm/activities/index.js +5 -0
  168. package/lib/esm/activities/index.js.map +1 -0
  169. package/lib/esm/activities/media/prepareAudio.js +200 -0
  170. package/lib/esm/activities/media/prepareAudio.js.map +1 -0
  171. package/lib/esm/activities/media/prepareVideo.js +390 -0
  172. package/lib/esm/activities/media/prepareVideo.js.map +1 -0
  173. package/lib/esm/activities/media/processPdfWithTextract.js +99 -0
  174. package/lib/esm/activities/media/processPdfWithTextract.js.map +1 -0
  175. package/lib/esm/activities/media/saveGladiaTranscription.js +78 -0
  176. package/lib/esm/activities/media/saveGladiaTranscription.js.map +1 -0
  177. package/lib/esm/activities/media/transcribeMediaWithGladia.js +79 -0
  178. package/lib/esm/activities/media/transcribeMediaWithGladia.js.map +1 -0
  179. package/lib/esm/activities/notifyWebhook.js +155 -0
  180. package/lib/esm/activities/notifyWebhook.js.map +1 -0
  181. package/lib/esm/activities/rateLimiter.js +27 -0
  182. package/lib/esm/activities/rateLimiter.js.map +1 -0
  183. package/lib/esm/activities/renditions/generateImageRendition.js +63 -0
  184. package/lib/esm/activities/renditions/generateImageRendition.js.map +1 -0
  185. package/lib/esm/activities/renditions/generateVideoRendition.js +194 -0
  186. package/lib/esm/activities/renditions/generateVideoRendition.js.map +1 -0
  187. package/lib/esm/activities/setDocumentStatus.js +12 -0
  188. package/lib/esm/activities/setDocumentStatus.js.map +1 -0
  189. package/lib/esm/conversion/TextractProcessor.js +410 -0
  190. package/lib/esm/conversion/TextractProcessor.js.map +1 -0
  191. package/lib/esm/conversion/image.js +143 -0
  192. package/lib/esm/conversion/image.js.map +1 -0
  193. package/lib/esm/conversion/markitdown.js +36 -0
  194. package/lib/esm/conversion/markitdown.js.map +1 -0
  195. package/lib/esm/conversion/mutool.js +139 -0
  196. package/lib/esm/conversion/mutool.js.map +1 -0
  197. package/lib/esm/conversion/pandoc.js +36 -0
  198. package/lib/esm/conversion/pandoc.js.map +1 -0
  199. package/lib/esm/dsl/conditions.js +75 -0
  200. package/lib/esm/dsl/conditions.js.map +1 -0
  201. package/lib/esm/dsl/dsl-workflow.js +336 -0
  202. package/lib/esm/dsl/dsl-workflow.js.map +1 -0
  203. package/lib/esm/dsl/dslProxyActivities.js +20 -0
  204. package/lib/esm/dsl/dslProxyActivities.js.map +1 -0
  205. package/lib/esm/dsl/projections.js +55 -0
  206. package/lib/esm/dsl/projections.js.map +1 -0
  207. package/lib/esm/dsl/setup/ActivityContext.js +117 -0
  208. package/lib/esm/dsl/setup/ActivityContext.js.map +1 -0
  209. package/lib/esm/dsl/setup/fetch/DataProvider.js +47 -0
  210. package/lib/esm/dsl/setup/fetch/DataProvider.js.map +1 -0
  211. package/lib/esm/dsl/setup/fetch/index.js +12 -0
  212. package/lib/esm/dsl/setup/fetch/index.js.map +1 -0
  213. package/lib/esm/dsl/setup/fetch/providers.js +61 -0
  214. package/lib/esm/dsl/setup/fetch/providers.js.map +1 -0
  215. package/lib/esm/dsl/test/test-child-workflow.js +5 -0
  216. package/lib/esm/dsl/test/test-child-workflow.js.map +1 -0
  217. package/lib/esm/dsl/validation.js +118 -0
  218. package/lib/esm/dsl/validation.js.map +1 -0
  219. package/lib/esm/dsl/vars.js +335 -0
  220. package/lib/esm/dsl/vars.js.map +1 -0
  221. package/lib/esm/dsl/walk.js +96 -0
  222. package/lib/esm/dsl/walk.js.map +1 -0
  223. package/lib/esm/dsl.js +4 -0
  224. package/lib/esm/dsl.js.map +1 -0
  225. package/lib/esm/errors.js +69 -0
  226. package/lib/esm/errors.js.map +1 -0
  227. package/lib/esm/index.js +38 -0
  228. package/lib/esm/index.js.map +1 -0
  229. package/lib/esm/iterative-generation/activities/extractToc.js +44 -0
  230. package/lib/esm/iterative-generation/activities/extractToc.js.map +1 -0
  231. package/lib/esm/iterative-generation/activities/finalizeOutput.js +69 -0
  232. package/lib/esm/iterative-generation/activities/finalizeOutput.js.map +1 -0
  233. package/lib/esm/iterative-generation/activities/generatePart.js +75 -0
  234. package/lib/esm/iterative-generation/activities/generatePart.js.map +1 -0
  235. package/lib/esm/iterative-generation/activities/generateToc.js +83 -0
  236. package/lib/esm/iterative-generation/activities/generateToc.js.map +1 -0
  237. package/lib/esm/iterative-generation/activities/index.js +5 -0
  238. package/lib/esm/iterative-generation/activities/index.js.map +1 -0
  239. package/lib/esm/iterative-generation/iterativeGenerationWorkflow.js +53 -0
  240. package/lib/esm/iterative-generation/iterativeGenerationWorkflow.js.map +1 -0
  241. package/lib/esm/iterative-generation/types.js +2 -0
  242. package/lib/esm/iterative-generation/types.js.map +1 -0
  243. package/lib/esm/iterative-generation/utils.js +112 -0
  244. package/lib/esm/iterative-generation/utils.js.map +1 -0
  245. package/lib/esm/result-types.js +7 -0
  246. package/lib/esm/result-types.js.map +1 -0
  247. package/lib/esm/system/notifyWebhookWorkflow.js +50 -0
  248. package/lib/esm/system/notifyWebhookWorkflow.js.map +1 -0
  249. package/lib/esm/system/recalculateEmbeddingsWorkflow.js +30 -0
  250. package/lib/esm/system/recalculateEmbeddingsWorkflow.js.map +1 -0
  251. package/lib/esm/utils/auth.js +8 -0
  252. package/lib/esm/utils/auth.js.map +1 -0
  253. package/lib/esm/utils/blobs.js +54 -0
  254. package/lib/esm/utils/blobs.js.map +1 -0
  255. package/lib/esm/utils/chunks.js +9 -0
  256. package/lib/esm/utils/chunks.js.map +1 -0
  257. package/lib/esm/utils/client.js +27 -0
  258. package/lib/esm/utils/client.js.map +1 -0
  259. package/lib/esm/utils/expand-vars.js +30 -0
  260. package/lib/esm/utils/expand-vars.js.map +1 -0
  261. package/lib/esm/utils/memory.js +55 -0
  262. package/lib/esm/utils/memory.js.map +1 -0
  263. package/lib/esm/utils/renditions.js +80 -0
  264. package/lib/esm/utils/renditions.js.map +1 -0
  265. package/lib/esm/utils/storage.js +45 -0
  266. package/lib/esm/utils/storage.js.map +1 -0
  267. package/lib/esm/utils/tokens.js +34 -0
  268. package/lib/esm/utils/tokens.js.map +1 -0
  269. package/lib/esm/vars.js +4 -0
  270. package/lib/esm/vars.js.map +1 -0
  271. package/lib/esm/workflows.js +8 -0
  272. package/lib/esm/workflows.js.map +1 -0
  273. package/lib/tsconfig.tsbuildinfo +1 -0
  274. package/lib/types/activities/advanced/createDocumentTypeFromInteractionRun.d.ts +17 -0
  275. package/lib/types/activities/advanced/createDocumentTypeFromInteractionRun.d.ts.map +1 -0
  276. package/lib/types/activities/advanced/createOrUpdateDocumentFromInteractionRun.d.ts +39 -0
  277. package/lib/types/activities/advanced/createOrUpdateDocumentFromInteractionRun.d.ts.map +1 -0
  278. package/lib/types/activities/advanced/updateDocumentFromInteractionRun.d.ts +19 -0
  279. package/lib/types/activities/advanced/updateDocumentFromInteractionRun.d.ts.map +1 -0
  280. package/lib/types/activities/chunkDocument.d.ts +33 -0
  281. package/lib/types/activities/chunkDocument.d.ts.map +1 -0
  282. package/lib/types/activities/copyParentArtifacts.d.ts +19 -0
  283. package/lib/types/activities/copyParentArtifacts.d.ts.map +1 -0
  284. package/lib/types/activities/createDocumentFromOther.d.ts +21 -0
  285. package/lib/types/activities/createDocumentFromOther.d.ts.map +1 -0
  286. package/lib/types/activities/executeInteraction.d.ts +61 -0
  287. package/lib/types/activities/executeInteraction.d.ts.map +1 -0
  288. package/lib/types/activities/extractDocumentText.d.ts +10 -0
  289. package/lib/types/activities/extractDocumentText.d.ts.map +1 -0
  290. package/lib/types/activities/generateDocumentProperties.d.ts +32 -0
  291. package/lib/types/activities/generateDocumentProperties.d.ts.map +1 -0
  292. package/lib/types/activities/generateEmbeddings.d.ts +53 -0
  293. package/lib/types/activities/generateEmbeddings.d.ts.map +1 -0
  294. package/lib/types/activities/generateOrAssignContentType.d.ts +44 -0
  295. package/lib/types/activities/generateOrAssignContentType.d.ts.map +1 -0
  296. package/lib/types/activities/getObjectFromStore.d.ts +14 -0
  297. package/lib/types/activities/getObjectFromStore.d.ts.map +1 -0
  298. package/lib/types/activities/handleError.d.ts +6 -0
  299. package/lib/types/activities/handleError.d.ts.map +1 -0
  300. package/lib/types/activities/index-dsl.d.ts +25 -0
  301. package/lib/types/activities/index-dsl.d.ts.map +1 -0
  302. package/lib/types/activities/index.d.ts +5 -0
  303. package/lib/types/activities/index.d.ts.map +1 -0
  304. package/lib/types/activities/media/prepareAudio.d.ts +25 -0
  305. package/lib/types/activities/media/prepareAudio.d.ts.map +1 -0
  306. package/lib/types/activities/media/prepareVideo.d.ts +30 -0
  307. package/lib/types/activities/media/prepareVideo.d.ts.map +1 -0
  308. package/lib/types/activities/media/processPdfWithTextract.d.ts +26 -0
  309. package/lib/types/activities/media/processPdfWithTextract.d.ts.map +1 -0
  310. package/lib/types/activities/media/saveGladiaTranscription.d.ts +14 -0
  311. package/lib/types/activities/media/saveGladiaTranscription.d.ts.map +1 -0
  312. package/lib/types/activities/media/transcribeMediaWithGladia.d.ts +19 -0
  313. package/lib/types/activities/media/transcribeMediaWithGladia.d.ts.map +1 -0
  314. package/lib/types/activities/notifyWebhook.d.ts +27 -0
  315. package/lib/types/activities/notifyWebhook.d.ts.map +1 -0
  316. package/lib/types/activities/rateLimiter.d.ts +11 -0
  317. package/lib/types/activities/rateLimiter.d.ts.map +1 -0
  318. package/lib/types/activities/renditions/generateImageRendition.d.ts +14 -0
  319. package/lib/types/activities/renditions/generateImageRendition.d.ts.map +1 -0
  320. package/lib/types/activities/renditions/generateVideoRendition.d.ts +15 -0
  321. package/lib/types/activities/renditions/generateVideoRendition.d.ts.map +1 -0
  322. package/lib/types/activities/setDocumentStatus.d.ts +15 -0
  323. package/lib/types/activities/setDocumentStatus.d.ts.map +1 -0
  324. package/lib/types/conversion/TextractProcessor.d.ts +45 -0
  325. package/lib/types/conversion/TextractProcessor.d.ts.map +1 -0
  326. package/lib/types/conversion/image.d.ts +13 -0
  327. package/lib/types/conversion/image.d.ts.map +1 -0
  328. package/lib/types/conversion/markitdown.d.ts +2 -0
  329. package/lib/types/conversion/markitdown.d.ts.map +1 -0
  330. package/lib/types/conversion/mutool.d.ts +19 -0
  331. package/lib/types/conversion/mutool.d.ts.map +1 -0
  332. package/lib/types/conversion/pandoc.d.ts +2 -0
  333. package/lib/types/conversion/pandoc.d.ts.map +1 -0
  334. package/lib/types/dsl/conditions.d.ts +2 -0
  335. package/lib/types/dsl/conditions.d.ts.map +1 -0
  336. package/lib/types/dsl/dsl-workflow.d.ts +5 -0
  337. package/lib/types/dsl/dsl-workflow.d.ts.map +1 -0
  338. package/lib/types/dsl/dslProxyActivities.d.ts +10 -0
  339. package/lib/types/dsl/dslProxyActivities.d.ts.map +1 -0
  340. package/lib/types/dsl/projections.d.ts +4 -0
  341. package/lib/types/dsl/projections.d.ts.map +1 -0
  342. package/lib/types/dsl/setup/ActivityContext.d.ts +17 -0
  343. package/lib/types/dsl/setup/ActivityContext.d.ts.map +1 -0
  344. package/lib/types/dsl/setup/fetch/DataProvider.d.ts +9 -0
  345. package/lib/types/dsl/setup/fetch/DataProvider.d.ts.map +1 -0
  346. package/lib/types/dsl/setup/fetch/index.d.ts +6 -0
  347. package/lib/types/dsl/setup/fetch/index.d.ts.map +1 -0
  348. package/lib/types/dsl/setup/fetch/providers.d.ts +25 -0
  349. package/lib/types/dsl/setup/fetch/providers.d.ts.map +1 -0
  350. package/lib/types/dsl/test/test-child-workflow.d.ts +4 -0
  351. package/lib/types/dsl/test/test-child-workflow.d.ts.map +1 -0
  352. package/lib/types/dsl/validation.d.ts +4 -0
  353. package/lib/types/dsl/validation.d.ts.map +1 -0
  354. package/lib/types/dsl/vars.d.ts +48 -0
  355. package/lib/types/dsl/vars.d.ts.map +1 -0
  356. package/lib/types/dsl/walk.d.ts +18 -0
  357. package/lib/types/dsl/walk.d.ts.map +1 -0
  358. package/lib/types/dsl.d.ts +4 -0
  359. package/lib/types/dsl.d.ts.map +1 -0
  360. package/lib/types/errors.d.ts +37 -0
  361. package/lib/types/errors.d.ts.map +1 -0
  362. package/lib/types/index.d.ts +37 -0
  363. package/lib/types/index.d.ts.map +1 -0
  364. package/lib/types/iterative-generation/activities/extractToc.d.ts +10 -0
  365. package/lib/types/iterative-generation/activities/extractToc.d.ts.map +1 -0
  366. package/lib/types/iterative-generation/activities/finalizeOutput.d.ts +3 -0
  367. package/lib/types/iterative-generation/activities/finalizeOutput.d.ts.map +1 -0
  368. package/lib/types/iterative-generation/activities/generatePart.d.ts +3 -0
  369. package/lib/types/iterative-generation/activities/generatePart.d.ts.map +1 -0
  370. package/lib/types/iterative-generation/activities/generateToc.d.ts +4 -0
  371. package/lib/types/iterative-generation/activities/generateToc.d.ts.map +1 -0
  372. package/lib/types/iterative-generation/activities/index.d.ts +5 -0
  373. package/lib/types/iterative-generation/activities/index.d.ts.map +1 -0
  374. package/lib/types/iterative-generation/iterativeGenerationWorkflow.d.ts +3 -0
  375. package/lib/types/iterative-generation/iterativeGenerationWorkflow.d.ts.map +1 -0
  376. package/lib/types/iterative-generation/types.d.ts +79 -0
  377. package/lib/types/iterative-generation/types.d.ts.map +1 -0
  378. package/lib/types/iterative-generation/utils.d.ts +26 -0
  379. package/lib/types/iterative-generation/utils.d.ts.map +1 -0
  380. package/lib/types/result-types.d.ts +22 -0
  381. package/lib/types/result-types.d.ts.map +1 -0
  382. package/lib/types/system/notifyWebhookWorkflow.d.ts +8 -0
  383. package/lib/types/system/notifyWebhookWorkflow.d.ts.map +1 -0
  384. package/lib/types/system/recalculateEmbeddingsWorkflow.d.ts +25 -0
  385. package/lib/types/system/recalculateEmbeddingsWorkflow.d.ts.map +1 -0
  386. package/lib/types/utils/auth.d.ts +4 -0
  387. package/lib/types/utils/auth.d.ts.map +1 -0
  388. package/lib/types/utils/blobs.d.ts +7 -0
  389. package/lib/types/utils/blobs.d.ts.map +1 -0
  390. package/lib/types/utils/chunks.d.ts +9 -0
  391. package/lib/types/utils/chunks.d.ts.map +1 -0
  392. package/lib/types/utils/client.d.ts +8 -0
  393. package/lib/types/utils/client.d.ts.map +1 -0
  394. package/lib/types/utils/expand-vars.d.ts +8 -0
  395. package/lib/types/utils/expand-vars.d.ts.map +1 -0
  396. package/lib/types/utils/memory.d.ts +8 -0
  397. package/lib/types/utils/memory.d.ts.map +1 -0
  398. package/lib/types/utils/renditions.d.ts +23 -0
  399. package/lib/types/utils/renditions.d.ts.map +1 -0
  400. package/lib/types/utils/storage.d.ts +16 -0
  401. package/lib/types/utils/storage.d.ts.map +1 -0
  402. package/lib/types/utils/tokens.d.ts +11 -0
  403. package/lib/types/utils/tokens.d.ts.map +1 -0
  404. package/lib/types/vars.d.ts +3 -0
  405. package/lib/types/vars.d.ts.map +1 -0
  406. package/lib/types/workflows.d.ts +8 -0
  407. package/lib/types/workflows.d.ts.map +1 -0
  408. package/lib/workflows-bundle.js +17213 -0
  409. package/package.json +146 -0
  410. package/src/activities/advanced/createDocumentTypeFromInteractionRun.ts +55 -0
  411. package/src/activities/advanced/createOrUpdateDocumentFromInteractionRun.ts +119 -0
  412. package/src/activities/advanced/updateDocumentFromInteractionRun.ts +35 -0
  413. package/src/activities/chunkDocument.ts +146 -0
  414. package/src/activities/copyParentArtifacts.ts +162 -0
  415. package/src/activities/createDocumentFromOther.ts +92 -0
  416. package/src/activities/executeInteraction.ts +300 -0
  417. package/src/activities/extractDocumentText.ts +205 -0
  418. package/src/activities/generateDocumentProperties.ts +120 -0
  419. package/src/activities/generateEmbeddings.ts +387 -0
  420. package/src/activities/generateOrAssignContentType.ts +218 -0
  421. package/src/activities/getObjectFromStore.ts +31 -0
  422. package/src/activities/handleError.ts +25 -0
  423. package/src/activities/index-dsl.ts +25 -0
  424. package/src/activities/index.ts +4 -0
  425. package/src/activities/media/prepareAudio.ts +334 -0
  426. package/src/activities/media/prepareVideo.ts +622 -0
  427. package/src/activities/media/processPdfWithTextract.ts +141 -0
  428. package/src/activities/media/saveGladiaTranscription.ts +128 -0
  429. package/src/activities/media/transcribeMediaWithGladia.ts +117 -0
  430. package/src/activities/notifyWebhook.test.ts +134 -0
  431. package/src/activities/notifyWebhook.ts +199 -0
  432. package/src/activities/rateLimiter.ts +41 -0
  433. package/src/activities/renditions/generateImageRendition.ts +111 -0
  434. package/src/activities/renditions/generateVideoRendition.ts +293 -0
  435. package/src/activities/setDocumentStatus.ts +25 -0
  436. package/src/conversion/TextractProcessor.ts +506 -0
  437. package/src/conversion/image.test.ts +118 -0
  438. package/src/conversion/image.ts +168 -0
  439. package/src/conversion/markitdown.ts +41 -0
  440. package/src/conversion/mutool.test.ts +74 -0
  441. package/src/conversion/mutool.ts +180 -0
  442. package/src/conversion/pandoc.test.ts +24 -0
  443. package/src/conversion/pandoc.ts +40 -0
  444. package/src/dsl/conditions.ts +76 -0
  445. package/src/dsl/dsl-workflow.test.ts +58 -0
  446. package/src/dsl/dsl-workflow.ts +397 -0
  447. package/src/dsl/dslProxyActivities.ts +38 -0
  448. package/src/dsl/ms.d.ts +11 -0
  449. package/src/dsl/projections.test.ts +159 -0
  450. package/src/dsl/projections.ts +72 -0
  451. package/src/dsl/setup/ActivityContext.ts +178 -0
  452. package/src/dsl/setup/fetch/DataProvider.ts +45 -0
  453. package/src/dsl/setup/fetch/index.ts +19 -0
  454. package/src/dsl/setup/fetch/providers.ts +67 -0
  455. package/src/dsl/test/test-child-workflow.ts +6 -0
  456. package/src/dsl/validation.test.ts +257 -0
  457. package/src/dsl/validation.ts +125 -0
  458. package/src/dsl/vars.test.ts +245 -0
  459. package/src/dsl/vars.ts +340 -0
  460. package/src/dsl/walk.test.ts +81 -0
  461. package/src/dsl/walk.ts +103 -0
  462. package/src/dsl/workflow-exec-child.test.ts +273 -0
  463. package/src/dsl/workflow-fetch.test.ts +138 -0
  464. package/src/dsl/workflow-import.test.ts +89 -0
  465. package/src/dsl/workflow.test.ts +122 -0
  466. package/src/dsl.ts +3 -0
  467. package/src/errors.ts +101 -0
  468. package/src/index.ts +41 -0
  469. package/src/iterative-generation/activities/extractToc.ts +63 -0
  470. package/src/iterative-generation/activities/finalizeOutput.ts +100 -0
  471. package/src/iterative-generation/activities/generatePart.ts +123 -0
  472. package/src/iterative-generation/activities/generateToc.ts +116 -0
  473. package/src/iterative-generation/activities/index.ts +4 -0
  474. package/src/iterative-generation/iterativeGenerationWorkflow.ts +68 -0
  475. package/src/iterative-generation/types.ts +99 -0
  476. package/src/iterative-generation/utils.ts +126 -0
  477. package/src/result-types.ts +25 -0
  478. package/src/system/notifyWebhookWorkflow.ts +70 -0
  479. package/src/system/recalculateEmbeddingsWorkflow.ts +41 -0
  480. package/src/utils/auth.ts +10 -0
  481. package/src/utils/blobs.ts +59 -0
  482. package/src/utils/chunks.ts +17 -0
  483. package/src/utils/client.ts +46 -0
  484. package/src/utils/expand-vars.ts +31 -0
  485. package/src/utils/memory.ts +61 -0
  486. package/src/utils/renditions.ts +127 -0
  487. package/src/utils/storage.ts +60 -0
  488. package/src/utils/tokens.ts +44 -0
  489. package/src/vars.ts +3 -0
  490. package/src/workflows.ts +7 -0
@@ -0,0 +1,194 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.executeInteraction = executeInteraction;
4
+ exports.executeInteractionFromActivity = executeInteractionFromActivity;
5
+ const activity_1 = require("@temporalio/activity");
6
+ const node_1 = require("@vertesia/client/node");
7
+ const common_1 = require("@vertesia/common");
8
+ const projections_js_1 = require("../dsl/projections.js");
9
+ const ActivityContext_js_1 = require("../dsl/setup/ActivityContext.js");
10
+ const errors_js_1 = require("../errors.js");
11
+ const tokens_js_1 = require("../utils/tokens.js");
12
+ const stream_1 = require("stream");
13
+ //Example:
14
+ //@ts-ignore
15
+ const JSON = {
16
+ name: "executeInteraction",
17
+ import: ["defaultModel", "guidlineId", "docTypeId"],
18
+ params: {
19
+ defaultModel: "${model}",
20
+ interactionName: "GenerateSummary",
21
+ model: "${defaultModel ?? 'gpt4'}",
22
+ environment: "13456",
23
+ max_tokens: 100,
24
+ temperature: 0.5,
25
+ tags: ["test"],
26
+ result_schema: "${docType.object_schema}",
27
+ prompt_data: {
28
+ documents: "${documents}",
29
+ guidline: "${guidline.text}",
30
+ },
31
+ },
32
+ fetch: {
33
+ documents: {
34
+ type: "document",
35
+ query: {
36
+ id: { $in: "${objectIds}" },
37
+ },
38
+ select: "+text",
39
+ },
40
+ guidline: {
41
+ type: "document",
42
+ limit: 1,
43
+ query: {
44
+ id: "${guidlineId}",
45
+ },
46
+ select: "+text",
47
+ on_not_found: "throw",
48
+ },
49
+ docType: {
50
+ type: "document_type",
51
+ limit: 1,
52
+ query: {
53
+ id: "${docTypeId}",
54
+ },
55
+ select: "+object_schema",
56
+ },
57
+ },
58
+ };
59
+ async function executeInteraction(payload) {
60
+ const { client, params } = await (0, ActivityContext_js_1.setupActivity)(payload);
61
+ const { interactionName, prompt_data, static_prompt_data: wf_prompt_data } = params;
62
+ if (wf_prompt_data) {
63
+ Object.assign(prompt_data, wf_prompt_data);
64
+ }
65
+ if (!interactionName) {
66
+ activity_1.log.error("Missing interactionName", { params });
67
+ throw new errors_js_1.ActivityParamNotFoundError("interactionName", payload.activity);
68
+ }
69
+ if (params.truncate) {
70
+ const truncate = params.truncate;
71
+ for (const [key, value] of Object.entries(truncate)) {
72
+ prompt_data[key] = (0, tokens_js_1.truncByMaxTokens)(prompt_data[key], value);
73
+ }
74
+ }
75
+ try {
76
+ const res = await executeInteractionFromActivity(client, interactionName, params, prompt_data, payload.debug_mode);
77
+ let completionResult = res.result;
78
+ // Handle image uploads if the result contains base64 images
79
+ const imageResults = completionResult.filter(r => r.type === "image");
80
+ if (imageResults.length > 0) {
81
+ const uploadedImages = await Promise.all(completionResult.map(async (item, index) => {
82
+ if (item.type === "image") {
83
+ const image = item.value;
84
+ // Extract base64 data and create buffer
85
+ const base64Data = image.replace(/^data:image\/[a-z]+;base64,/, "");
86
+ const buffer = Buffer.from(base64Data, 'base64');
87
+ // Generate filename
88
+ const { runId } = (0, activity_1.activityInfo)().workflowExecution;
89
+ const { activityId } = (0, activity_1.activityInfo)();
90
+ const filename = `generated-image-${runId}-${activityId}-${index}.png`;
91
+ // Create a readable stream from the buffer
92
+ const stream = stream_1.Readable.from(buffer);
93
+ const source = new node_1.NodeStreamSource(stream, filename, "image/png");
94
+ const file = await client.files.uploadFile(source);
95
+ return { type: "image", value: file };
96
+ }
97
+ return item;
98
+ }));
99
+ completionResult = uploadedImages;
100
+ }
101
+ return (0, projections_js_1.projectResult)(payload, params, res, {
102
+ runId: res.id,
103
+ status: res.status,
104
+ result: completionResult,
105
+ });
106
+ }
107
+ catch (error) {
108
+ activity_1.log.error(`Failed to execute interaction ${interactionName}`, { error });
109
+ if (error.statusCode === 429 && params.exit_on_resource_exhaustion) {
110
+ throw new errors_js_1.ResourceExhaustedError(error.statusCode, "Resource exhausted - rate limit exceeded");
111
+ }
112
+ else if (error.message.includes("Failed to validate merged prompt schema")) {
113
+ //issue with the input data, don't retry
114
+ throw new errors_js_1.ActivityParamInvalidError("prompt_data", payload.activity, error.message);
115
+ }
116
+ else if (error.message.includes("modelId: Path `modelId` is required")) {
117
+ //issue with the input data, don't retry
118
+ throw new errors_js_1.ActivityParamInvalidError("model", payload.activity, error.message);
119
+ }
120
+ else {
121
+ throw new Error(`Interaction Execution failed ${interactionName}: ${error.message}`);
122
+ }
123
+ }
124
+ }
125
+ async function executeInteractionFromActivity(client, interactionName, params, prompt_data, debug) {
126
+ const userTags = params.tags;
127
+ const info = (0, activity_1.activityInfo)();
128
+ const runId = info.workflowExecution.runId;
129
+ let tags = ["workflow"];
130
+ if (userTags) {
131
+ tags = tags.concat(userTags);
132
+ }
133
+ const workflow = {
134
+ run_id: info.workflowExecution.runId,
135
+ workflow_id: info.workflowExecution.workflowId,
136
+ activity_type: info.activityType,
137
+ };
138
+ let previousStudioExecutionRun = undefined;
139
+ if (params.include_previous_error) {
140
+ //retrieve last failed run if any
141
+ if (info.attempt > 1) {
142
+ activity_1.log.info("Retrying, searching for previous run", { prev_run_id: runId });
143
+ const payload = {
144
+ query: { workflow_run_ids: [runId] },
145
+ limit: 1,
146
+ };
147
+ const previousRun = await client.runs.search(payload).then((res) => {
148
+ activity_1.log.info("Search results", { results: res });
149
+ return res ? (res[0] ?? undefined) : undefined;
150
+ });
151
+ if (previousRun) {
152
+ activity_1.log.info("Found previous run", { previousRun });
153
+ previousStudioExecutionRun = await client.runs.retrieve(previousRun.id);
154
+ }
155
+ }
156
+ }
157
+ if (debug && previousStudioExecutionRun?.error) {
158
+ activity_1.log.info(`Found previous run error`, { error: previousStudioExecutionRun?.error });
159
+ }
160
+ const config = {
161
+ environment: params.environment,
162
+ model: params.model,
163
+ model_options: params.model_options,
164
+ do_validate: params.validate_result,
165
+ };
166
+ const data = {
167
+ ...prompt_data,
168
+ previous_error: previousStudioExecutionRun?.error,
169
+ };
170
+ const result_schema = params.result_schema;
171
+ activity_1.log.debug(`About to execute interaction ${interactionName}`, { config, data, result_schema, tags, workflow });
172
+ const res = await client.interactions
173
+ .executeByName(interactionName, {
174
+ config,
175
+ data,
176
+ result_schema,
177
+ tags,
178
+ stream: false,
179
+ workflow,
180
+ })
181
+ .catch((err) => {
182
+ activity_1.log.error(`Error executing interaction ${interactionName}`, { err });
183
+ throw err;
184
+ });
185
+ if (debug) {
186
+ activity_1.log.info(`Interaction executed ${interactionName}`, res);
187
+ }
188
+ if (res.error || res.status === common_1.ExecutionRunStatus.failed) {
189
+ activity_1.log.error(`Error executing interaction ${interactionName}`, { error: res.error });
190
+ throw new Error(`Interaction Execution failed ${interactionName}: ${res.error}`);
191
+ }
192
+ return res;
193
+ }
194
+ //# sourceMappingURL=executeInteraction.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"executeInteraction.js","sourceRoot":"","sources":["../../../src/activities/executeInteraction.ts"],"names":[],"mappings":";;AAiIA,gDAqFC;AAED,wEAmFC;AA1SD,mDAAyD;AAEzD,gDAAyD;AACzD,6CAQ0B;AAC1B,0DAAsD;AACtD,wEAAgE;AAChE,4CAA6G;AAC7G,kDAAoE;AACpE,mCAAkC;AAElC,UAAU;AACV,YAAY;AACZ,MAAM,IAAI,GAAoB;IAC1B,IAAI,EAAE,oBAAoB;IAC1B,MAAM,EAAE,CAAC,cAAc,EAAE,YAAY,EAAE,WAAW,CAAC;IACnD,MAAM,EAAE;QACJ,YAAY,EAAE,UAAU;QACxB,eAAe,EAAE,iBAAiB;QAClC,KAAK,EAAE,2BAA2B;QAClC,WAAW,EAAE,OAAO;QACpB,UAAU,EAAE,GAAG;QACf,WAAW,EAAE,GAAG;QAChB,IAAI,EAAE,CAAC,MAAM,CAAC;QACd,aAAa,EAAE,0BAA0B;QACzC,WAAW,EAAE;YACT,SAAS,EAAE,cAAc;YACzB,QAAQ,EAAE,kBAAkB;SAC/B;KACJ;IACD,KAAK,EAAE;QACH,SAAS,EAAE;YACP,IAAI,EAAE,UAAU;YAChB,KAAK,EAAE;gBACH,EAAE,EAAE,EAAE,GAAG,EAAE,cAAc,EAAE;aAC9B;YACD,MAAM,EAAE,OAAO;SAClB;QACD,QAAQ,EAAE;YACN,IAAI,EAAE,UAAU;YAChB,KAAK,EAAE,CAAC;YACR,KAAK,EAAE;gBACH,EAAE,EAAE,eAAe;aACtB;YACD,MAAM,EAAE,OAAO;YACf,YAAY,EAAE,OAAO;SACxB;QACD,OAAO,EAAE;YACL,IAAI,EAAE,eAAe;YACrB,KAAK,EAAE,CAAC;YACR,KAAK,EAAE;gBACH,EAAE,EAAE,cAAc;aACrB;YACD,MAAM,EAAE,gBAAgB;SAC3B;KACJ;CACJ,CAAC;AAiEK,KAAK,UAAU,kBAAkB,CAAC,OAA8D;IACnG,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,IAAA,kCAAa,EAA2B,OAAO,CAAC,CAAC;IAElF,MAAM,EAAE,eAAe,EAAE,WAAW,EAAE,kBAAkB,EAAE,cAAc,EAAE,GAAG,MAAM,CAAC;IACpF,IAAI,cAAc,EAAE,CAAC;QACjB,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;IAC/C,CAAC;IAED,IAAI,CAAC,eAAe,EAAE,CAAC;QACnB,cAAG,CAAC,KAAK,CAAC,yBAAyB,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;QACjD,MAAM,IAAI,sCAA0B,CAAC,iBAAiB,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC9E,CAAC;IAED,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;QAClB,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;QACjC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;YAClD,WAAW,CAAC,GAAG,CAAC,GAAG,IAAA,4BAAgB,EAAC,WAAW,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,CAAC;QACjE,CAAC;IACL,CAAC;IAED,IAAI,CAAC;QACD,MAAM,GAAG,GAAG,MAAM,8BAA8B,CAC5C,MAAM,EACN,eAAe,EACf,MAAM,EACN,WAAW,EACX,OAAO,CAAC,UAAU,CACrB,CAAC;QAEF,IAAI,gBAAgB,GAAuB,GAAG,CAAC,MAAM,CAAC;QAEtD,4DAA4D;QAC5D,MAAM,YAAY,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC;QACtE,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1B,MAAM,cAAc,GAAG,MAAM,OAAO,CAAC,GAAG,CACpC,gBAAgB,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE;gBACvC,IAAI,IAAI,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;oBACxB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;oBACzB,wCAAwC;oBACxC,MAAM,UAAU,GAAG,KAAK,CAAC,OAAO,CAAC,6BAA6B,EAAE,EAAE,CAAC,CAAC;oBACpE,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;oBAEjD,oBAAoB;oBACpB,MAAM,EAAE,KAAK,EAAE,GAAG,IAAA,uBAAY,GAAE,CAAC,iBAAiB,CAAC;oBACnD,MAAM,EAAE,UAAU,EAAE,GAAG,IAAA,uBAAY,GAAE,CAAC;oBACtC,MAAM,QAAQ,GAAG,mBAAmB,KAAK,IAAI,UAAU,IAAI,KAAK,MAAM,CAAC;oBAEvE,2CAA2C;oBAC3C,MAAM,MAAM,GAAG,iBAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;oBAErC,MAAM,MAAM,GAAG,IAAI,uBAAgB,CAC/B,MAAM,EACN,QAAQ,EACR,WAAW,CACd,CAAC;oBAEF,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;oBACnD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAsB,CAAC;gBAC9D,CAAC;gBACD,OAAO,IAAI,CAAC;YAChB,CAAC,CAAC,CACL,CAAC;YACF,gBAAgB,GAAG,cAAc,CAAC;QACtC,CAAC;QAED,OAAO,IAAA,8BAAa,EAAC,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE;YACvC,KAAK,EAAE,GAAG,CAAC,EAAE;YACb,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,MAAM,EAAE,gBAAgB;SAC3B,CAAC,CAAC;IAEP,CAAC;IAAC,OAAO,KAAU,EAAE,CAAC;QAClB,cAAG,CAAC,KAAK,CAAC,iCAAiC,eAAe,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QACzE,IAAI,KAAK,CAAC,UAAU,KAAK,GAAG,IAAI,MAAM,CAAC,2BAA2B,EAAE,CAAC;YACjE,MAAM,IAAI,kCAAsB,CAAC,KAAK,CAAC,UAAU,EAAE,0CAA0C,CAAC,CAAC;QACnG,CAAC;aAAM,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,yCAAyC,CAAC,EAAE,CAAC;YAC3E,wCAAwC;YACxC,MAAM,IAAI,qCAAyB,CAAC,aAAa,EAAE,OAAO,CAAC,QAAQ,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;QACxF,CAAC;aAAM,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,qCAAqC,CAAC,EAAE,CAAC;YACvE,wCAAwC;YACxC,MAAM,IAAI,qCAAyB,CAAC,OAAO,EAAE,OAAO,CAAC,QAAQ,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;QAClF,CAAC;aAAM,CAAC;YACJ,MAAM,IAAI,KAAK,CAAC,gCAAgC,eAAe,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QACzF,CAAC;IACL,CAAC;AACL,CAAC;AAEM,KAAK,UAAU,8BAA8B,CAChD,MAAsB,EACtB,eAAuB,EACvB,MAAkC,EAClC,WAAgB,EAChB,KAAe;IAEf,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC;IAC7B,MAAM,IAAI,GAAG,IAAA,uBAAY,GAAE,CAAC;IAC5B,MAAM,KAAK,GAAG,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC;IAC3C,IAAI,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC;IACxB,IAAI,QAAQ,EAAE,CAAC;QACX,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IACjC,CAAC;IACD,MAAM,QAAQ,GAAyB;QACnC,MAAM,EAAE,IAAI,CAAC,iBAAiB,CAAC,KAAK;QACpC,WAAW,EAAE,IAAI,CAAC,iBAAiB,CAAC,UAAU;QAC9C,aAAa,EAAE,IAAI,CAAC,YAAY;KACnC,CAAC;IAEF,IAAI,0BAA0B,GAA6B,SAAS,CAAC;IACrE,IAAI,MAAM,CAAC,sBAAsB,EAAE,CAAC;QAChC,iCAAiC;QACjC,IAAI,IAAI,CAAC,OAAO,GAAG,CAAC,EAAE,CAAC;YACnB,cAAG,CAAC,IAAI,CAAC,sCAAsC,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;YACzE,MAAM,OAAO,GAAqB;gBAC9B,KAAK,EAAE,EAAE,gBAAgB,EAAE,CAAC,KAAK,CAAC,EAAE;gBACpC,KAAK,EAAE,CAAC;aACX,CAAC;YACF,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE;gBAC/D,cAAG,CAAC,IAAI,CAAC,gBAAgB,EAAE,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC;gBAC7C,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YACnD,CAAC,CAAC,CAAC;YAEH,IAAI,WAAW,EAAE,CAAC;gBACd,cAAG,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC;gBAChD,0BAA0B,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;YAC5E,CAAC;QACL,CAAC;IACL,CAAC;IACD,IAAI,KAAK,IAAI,0BAA0B,EAAE,KAAK,EAAE,CAAC;QAC7C,cAAG,CAAC,IAAI,CAAC,2BAA2B,EAAE,EAAE,KAAK,EAAE,0BAA0B,EAAE,KAAK,EAAE,CAAC,CAAC;IACxF,CAAC;IAED,MAAM,MAAM,GAAsC;QAC9C,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,aAAa,EAAE,MAAM,CAAC,aAAa;QACnC,WAAW,EAAE,MAAM,CAAC,eAAe;KACtC,CAAC;IACF,MAAM,IAAI,GAAG;QACT,GAAG,WAAW;QACd,cAAc,EAAE,0BAA0B,EAAE,KAAK;KACpD,CAAC;IAEF,MAAM,aAAa,GAAG,MAAM,CAAC,aAAa,CAAC;IAE3C,cAAG,CAAC,KAAK,CAAC,gCAAgC,eAAe,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;IAE9G,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,YAAY;SAChC,aAAa,CAAC,eAAe,EAAE;QAC5B,MAAM;QACN,IAAI;QACJ,aAAa;QACb,IAAI;QACJ,MAAM,EAAE,KAAK;QACb,QAAQ;KACX,CAAC;SACD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;QACX,cAAG,CAAC,KAAK,CAAC,+BAA+B,eAAe,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;QACrE,MAAM,GAAG,CAAC;IACd,CAAC,CAAC,CAAC;IAEP,IAAI,KAAK,EAAE,CAAC;QACR,cAAG,CAAC,IAAI,CAAC,wBAAwB,eAAe,EAAE,EAAE,GAAG,CAAC,CAAC;IAC7D,CAAC;IAED,IAAI,GAAG,CAAC,KAAK,IAAI,GAAG,CAAC,MAAM,KAAK,2BAAkB,CAAC,MAAM,EAAE,CAAC;QACxD,cAAG,CAAC,KAAK,CAAC,+BAA+B,eAAe,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;QAClF,MAAM,IAAI,KAAK,CAAC,gCAAgC,eAAe,KAAK,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;IACrF,CAAC;IAED,OAAO,GAAG,CAAC;AACf,CAAC"}
@@ -0,0 +1,156 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.extractDocumentText = extractDocumentText;
4
+ const activity_1 = require("@temporalio/activity");
5
+ const markitdown_js_1 = require("../conversion/markitdown.js");
6
+ const mutool_js_1 = require("../conversion/mutool.js");
7
+ const pandoc_js_1 = require("../conversion/pandoc.js");
8
+ const ActivityContext_js_1 = require("../dsl/setup/ActivityContext.js");
9
+ const errors_js_1 = require("../errors.js");
10
+ const result_types_js_1 = require("../result-types.js");
11
+ const blobs_js_1 = require("../utils/blobs.js");
12
+ const tokens_js_1 = require("../utils/tokens.js");
13
+ //@ts-ignore
14
+ const JSON = {
15
+ name: "extractDocumentText",
16
+ };
17
+ async function extractDocumentText(payload) {
18
+ const { client, objectId } = await (0, ActivityContext_js_1.setupActivity)(payload);
19
+ const r = await client.objects.find({
20
+ query: { _id: objectId },
21
+ limit: 1,
22
+ select: "+text",
23
+ });
24
+ const doc = r[0];
25
+ if (!doc) {
26
+ activity_1.log.error(`Document ${objectId} not found`);
27
+ throw new errors_js_1.DocumentNotFoundError(`Document ${objectId} not found`, payload.objectIds);
28
+ }
29
+ activity_1.log.info(`Extracting text for object ${doc.id}`);
30
+ if (!doc.content?.type || !doc.content?.source) {
31
+ if (doc.text) {
32
+ return createResponse(doc, doc.text, result_types_js_1.TextExtractionStatus.skipped, "Text present and no source or type");
33
+ }
34
+ else {
35
+ return createResponse(doc, "", result_types_js_1.TextExtractionStatus.error, "No source or type found");
36
+ }
37
+ }
38
+ //skip if text already extracted and proper etag
39
+ if (doc.text && doc.text.length > 0 && doc.text_etag === doc.content.etag) {
40
+ return createResponse(doc, doc.text, result_types_js_1.TextExtractionStatus.skipped, "Text already extracted");
41
+ }
42
+ let fileBuffer;
43
+ try {
44
+ fileBuffer = await (0, blobs_js_1.fetchBlobAsBuffer)(client, doc.content.source);
45
+ }
46
+ catch (e) {
47
+ activity_1.log.error(`Error reading file: ${e}`);
48
+ return createResponse(doc, "", result_types_js_1.TextExtractionStatus.error, e.message);
49
+ }
50
+ let txt;
51
+ switch (doc.content.type) {
52
+ case "application/pdf":
53
+ txt = await (0, mutool_js_1.mutoolPdfToText)(fileBuffer);
54
+ break;
55
+ case "text/plain":
56
+ txt = fileBuffer.toString("utf8");
57
+ break;
58
+ //docx
59
+ case "application/vnd.openxmlformats-officedocument.wordprocessingml.document":
60
+ txt = await (0, markitdown_js_1.markdownWithMarkitdown)(fileBuffer, "docx");
61
+ break;
62
+ //pptx
63
+ case "application/vnd.openxmlformats-officedocument.presentationml.presentation":
64
+ txt = await (0, markitdown_js_1.markdownWithMarkitdown)(fileBuffer, "pptx");
65
+ break;
66
+ //html
67
+ case "text/html":
68
+ txt = await (0, pandoc_js_1.markdownWithPandoc)(fileBuffer, "html");
69
+ break;
70
+ //opendocument
71
+ case "application/vnd.oasis.opendocument.text":
72
+ txt = await (0, pandoc_js_1.markdownWithPandoc)(fileBuffer, "odt");
73
+ break;
74
+ //rtf
75
+ case "application/rtf":
76
+ txt = await (0, pandoc_js_1.markdownWithPandoc)(fileBuffer, "rtf");
77
+ break;
78
+ //markdown
79
+ case "text/markdown":
80
+ txt = fileBuffer.toString("utf8");
81
+ break;
82
+ //csv
83
+ case "text/csv":
84
+ txt = fileBuffer.toString("utf8");
85
+ break;
86
+ //typescript
87
+ case "application/typescript":
88
+ txt = fileBuffer.toString("utf8");
89
+ break;
90
+ //javascript
91
+ case "application/javascript":
92
+ txt = fileBuffer.toString("utf8");
93
+ break;
94
+ //json
95
+ case "application/json":
96
+ txt = fileBuffer.toString("utf8");
97
+ break;
98
+ default:
99
+ if (sniffIfText(fileBuffer)) {
100
+ txt = fileBuffer.toString("utf8"); //TODO: add charset detection
101
+ break;
102
+ }
103
+ return createResponse(doc, doc.text ?? "", result_types_js_1.TextExtractionStatus.skipped, `Unsupported mime type: ${doc.content.type}`);
104
+ }
105
+ const tokensData = (0, tokens_js_1.countTokens)(txt);
106
+ const etag = doc.content.etag ?? (0, blobs_js_1.md5)(txt);
107
+ const updateData = {
108
+ text: txt,
109
+ text_etag: etag,
110
+ tokens: {
111
+ ...tokensData,
112
+ etag: etag,
113
+ },
114
+ };
115
+ await client.objects.update(doc.id, updateData);
116
+ return createResponse(doc, txt, result_types_js_1.TextExtractionStatus.success);
117
+ }
118
+ function createResponse(doc, text, status, message) {
119
+ return {
120
+ status,
121
+ message,
122
+ tokens: doc.tokens,
123
+ len: text.length,
124
+ objectId: doc.id,
125
+ hasText: !!text,
126
+ };
127
+ }
128
+ function sniffIfText(buf) {
129
+ // If file is too large, don't even try
130
+ if (buf.length > 500 * 1024) {
131
+ return false;
132
+ }
133
+ // Count binary/control characters
134
+ let binaryCount = 0;
135
+ const sampleSize = Math.min(buf.length, 1000); // Check first 1000 bytes
136
+ for (let i = 0; i < sampleSize; i++) {
137
+ // Count control characters (except common whitespace)
138
+ const byte = buf[i];
139
+ if ((byte < 32 && ![9, 10, 13].includes(byte)) || byte === 0) {
140
+ binaryCount++;
141
+ }
142
+ }
143
+ // If more than 10% binary/control chars, probably not text
144
+ if (binaryCount / sampleSize > 0.1) {
145
+ return false;
146
+ }
147
+ // Additional check for valid UTF-8 encoding
148
+ try {
149
+ const s = buf.toString("utf8");
150
+ return s.length > 0 && !s.includes("\uFFFD"); // Replacement character
151
+ }
152
+ catch (e) {
153
+ return false;
154
+ }
155
+ }
156
+ //# sourceMappingURL=extractDocumentText.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"extractDocumentText.js","sourceRoot":"","sources":["../../../src/activities/extractDocumentText.ts"],"names":[],"mappings":";;AA4BA,kDAgIC;AA5JD,mDAA2C;AAO3C,+DAAqE;AACrE,uDAA0D;AAC1D,uDAA6D;AAC7D,wEAAgE;AAChE,4CAAqD;AACrD,wDAAgF;AAChF,gDAA2D;AAC3D,kDAAiD;AAEjD,YAAY;AACZ,MAAM,IAAI,GAAoB;IAC1B,IAAI,EAAE,qBAAqB;CAC9B,CAAC;AASK,KAAK,UAAU,mBAAmB,CACrC,OAA+D;IAE/D,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAA,kCAAa,EAAC,OAAO,CAAC,CAAC;IAE1D,MAAM,CAAC,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC;QAChC,KAAK,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE;QACxB,KAAK,EAAE,CAAC;QACR,MAAM,EAAE,OAAO;KAClB,CAAC,CAAC;IACH,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,CAAkB,CAAC;IAClC,IAAI,CAAC,GAAG,EAAE,CAAC;QACP,cAAG,CAAC,KAAK,CAAC,YAAY,QAAQ,YAAY,CAAC,CAAC;QAC5C,MAAM,IAAI,iCAAqB,CAAC,YAAY,QAAQ,YAAY,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;IACzF,CAAC;IAED,cAAG,CAAC,IAAI,CAAC,8BAA8B,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC;IAEjD,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;QAC7C,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;YACX,OAAO,cAAc,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE,sCAAoB,CAAC,OAAO,EAAE,oCAAoC,CAAC,CAAC;QAC7G,CAAC;aAAM,CAAC;YACJ,OAAO,cAAc,CAAC,GAAG,EAAE,EAAE,EAAE,sCAAoB,CAAC,KAAK,EAAE,yBAAyB,CAAC,CAAC;QAC1F,CAAC;IACL,CAAC;IAED,gDAAgD;IAChD,IAAI,GAAG,CAAC,IAAI,IAAI,GAAG,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,GAAG,CAAC,SAAS,KAAK,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QACxE,OAAO,cAAc,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE,sCAAoB,CAAC,OAAO,EAAE,wBAAwB,CAAC,CAAC;IACjG,CAAC;IAED,IAAI,UAAkB,CAAC;IACvB,IAAI,CAAC;QACD,UAAU,GAAG,MAAM,IAAA,4BAAiB,EAAC,MAAM,EAAE,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACrE,CAAC;IAAC,OAAO,CAAM,EAAE,CAAC;QACd,cAAG,CAAC,KAAK,CAAC,uBAAuB,CAAC,EAAE,CAAC,CAAC;QACtC,OAAO,cAAc,CAAC,GAAG,EAAE,EAAE,EAAE,sCAAoB,CAAC,KAAK,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC;IAC1E,CAAC;IAED,IAAI,GAAW,CAAC;IAEhB,QAAQ,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QACvB,KAAK,iBAAiB;YAClB,GAAG,GAAG,MAAM,IAAA,2BAAe,EAAC,UAAU,CAAC,CAAC;YACxC,MAAM;QAEV,KAAK,YAAY;YACb,GAAG,GAAG,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAClC,MAAM;QAEV,MAAM;QACN,KAAK,yEAAyE;YAC1E,GAAG,GAAG,MAAM,IAAA,sCAAsB,EAAC,UAAU,EAAE,MAAM,CAAC,CAAC;YACvD,MAAM;QAEV,MAAM;QACN,KAAK,2EAA2E;YAC5E,GAAG,GAAG,MAAM,IAAA,sCAAsB,EAAC,UAAU,EAAE,MAAM,CAAC,CAAC;YACvD,MAAM;QAEV,MAAM;QACN,KAAK,WAAW;YACZ,GAAG,GAAG,MAAM,IAAA,8BAAkB,EAAC,UAAU,EAAE,MAAM,CAAC,CAAC;YACnD,MAAM;QAEV,cAAc;QACd,KAAK,yCAAyC;YAC1C,GAAG,GAAG,MAAM,IAAA,8BAAkB,EAAC,UAAU,EAAE,KAAK,CAAC,CAAC;YAClD,MAAM;QAEV,KAAK;QACL,KAAK,iBAAiB;YAClB,GAAG,GAAG,MAAM,IAAA,8BAAkB,EAAC,UAAU,EAAE,KAAK,CAAC,CAAC;YAClD,MAAM;QAEV,UAAU;QACV,KAAK,eAAe;YAChB,GAAG,GAAG,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAClC,MAAM;QAEV,KAAK;QACL,KAAK,UAAU;YACX,GAAG,GAAG,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAClC,MAAM;QAEV,YAAY;QACZ,KAAK,wBAAwB;YACzB,GAAG,GAAG,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAClC,MAAM;QAEV,YAAY;QACZ,KAAK,wBAAwB;YACzB,GAAG,GAAG,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAClC,MAAM;QAEV,MAAM;QACN,KAAK,kBAAkB;YACnB,GAAG,GAAG,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAClC,MAAM;QAEV;YACI,IAAI,WAAW,CAAC,UAAU,CAAC,EAAE,CAAC;gBAC1B,GAAG,GAAG,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,6BAA6B;gBAChE,MAAM;YACV,CAAC;YACD,OAAO,cAAc,CACjB,GAAG,EACH,GAAG,CAAC,IAAI,IAAI,EAAE,EACd,sCAAoB,CAAC,OAAO,EAC5B,0BAA0B,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAC/C,CAAC;IACV,CAAC;IAED,MAAM,UAAU,GAAG,IAAA,uBAAW,EAAC,GAAG,CAAC,CAAC;IACpC,MAAM,IAAI,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,IAAI,IAAA,cAAG,EAAC,GAAG,CAAC,CAAC;IAE1C,MAAM,UAAU,GAA+B;QAC3C,IAAI,EAAE,GAAG;QACT,SAAS,EAAE,IAAI;QACf,MAAM,EAAE;YACJ,GAAG,UAAU;YACb,IAAI,EAAE,IAAI;SACb;KACJ,CAAC;IAEF,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;IAEhD,OAAO,cAAc,CAAC,GAAG,EAAE,GAAG,EAAE,sCAAoB,CAAC,OAAO,CAAC,CAAC;AAClE,CAAC;AAED,SAAS,cAAc,CACnB,GAAkB,EAClB,IAAY,EACZ,MAA4B,EAC5B,OAAgB;IAEhB,OAAO;QACH,MAAM;QACN,OAAO;QACP,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,GAAG,EAAE,IAAI,CAAC,MAAM;QAChB,QAAQ,EAAE,GAAG,CAAC,EAAE;QAChB,OAAO,EAAE,CAAC,CAAC,IAAI;KAClB,CAAC;AACN,CAAC;AAED,SAAS,WAAW,CAAC,GAAW;IAC5B,uCAAuC;IACvC,IAAI,GAAG,CAAC,MAAM,GAAG,GAAG,GAAG,IAAI,EAAE,CAAC;QAC1B,OAAO,KAAK,CAAC;IACjB,CAAC;IAED,kCAAkC;IAClC,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,yBAAyB;IAExE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,sDAAsD;QACtD,MAAM,IAAI,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;QACpB,IAAI,CAAC,IAAI,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;YAC3D,WAAW,EAAE,CAAC;QAClB,CAAC;IACL,CAAC;IAED,2DAA2D;IAC3D,IAAI,WAAW,GAAG,UAAU,GAAG,GAAG,EAAE,CAAC;QACjC,OAAO,KAAK,CAAC;IACjB,CAAC;IAED,4CAA4C;IAC5C,IAAI,CAAC;QACD,MAAM,CAAC,GAAG,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAC/B,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,wBAAwB;IAC1E,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,OAAO,KAAK,CAAC;IACjB,CAAC;AACL,CAAC"}
@@ -0,0 +1,83 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.generateDocumentProperties = generateDocumentProperties;
4
+ const activity_1 = require("@temporalio/activity");
5
+ const ActivityContext_js_1 = require("../dsl/setup/ActivityContext.js");
6
+ const tokens_js_1 = require("../utils/tokens.js");
7
+ const executeInteraction_js_1 = require("./executeInteraction.js");
8
+ const INT_EXTRACT_INFORMATION = "sys:ExtractInformation";
9
+ async function generateDocumentProperties(payload) {
10
+ const context = await (0, ActivityContext_js_1.setupActivity)(payload);
11
+ const { params, client, objectId } = context;
12
+ const interactionName = params.interactionName ?? INT_EXTRACT_INFORMATION;
13
+ const project = await context.fetchProject();
14
+ const doc = await client.objects.retrieve(objectId, "+text");
15
+ const type = doc.type ? await client.types.retrieve(doc.type.id) : undefined;
16
+ if (!doc?.text && !params.use_vision && !doc?.content?.type?.startsWith("image/")) {
17
+ activity_1.log.warn(`Object ${objectId} not found or text is empty`);
18
+ return { status: "failed", error: "no-text" };
19
+ }
20
+ if (!type || !type.object_schema) {
21
+ activity_1.log.info(`Object ${objectId} has no schema`);
22
+ return { document: objectId, status: "skipped", message: "no schema defined on type" };
23
+ }
24
+ const getImageRef = () => {
25
+ if (doc.content?.type?.startsWith("image/")) {
26
+ return "store:" + doc.id;
27
+ }
28
+ if (params.use_vision && doc.content?.type?.startsWith("application/pdf")) {
29
+ return "store:" + doc.id;
30
+ }
31
+ activity_1.log.info(`Object ${objectId} is not an image or pdf`);
32
+ return undefined;
33
+ };
34
+ const content = doc.text
35
+ ? (0, tokens_js_1.truncByMaxTokens)(doc.text, params.truncate || 30000)
36
+ : undefined;
37
+ const promptData = {
38
+ content: content,
39
+ image: getImageRef() ?? undefined,
40
+ human_context: project?.configuration?.human_context ?? undefined,
41
+ };
42
+ activity_1.log.info(` Extracting information from object ${objectId} with type ${type.name}`, payload.debug_mode ? { params } : undefined);
43
+ const infoRes = await (0, executeInteraction_js_1.executeInteractionFromActivity)(client, interactionName, {
44
+ ...params,
45
+ include_previous_error: true,
46
+ result_schema: type.object_schema,
47
+ validate_result: type.strict_mode,
48
+ }, promptData, payload.debug_mode ?? false);
49
+ const getText = () => {
50
+ if (doc.text) {
51
+ return undefined;
52
+ }
53
+ let text = "";
54
+ const jsonResult = infoRes.result.object();
55
+ if (jsonResult.title) {
56
+ text += jsonResult.title + "\n";
57
+ }
58
+ if (jsonResult.description) {
59
+ text += jsonResult.description;
60
+ }
61
+ if (text) {
62
+ return text;
63
+ }
64
+ else {
65
+ return undefined;
66
+ }
67
+ };
68
+ activity_1.log.info(`Extracted information from object ${objectId} with type ${type.name}`, { runId: infoRes.id });
69
+ await client.objects.update(doc.id, {
70
+ properties: {
71
+ ...infoRes.result.object(),
72
+ etag: doc.text_etag,
73
+ },
74
+ text: getText(),
75
+ generation_run_info: {
76
+ id: infoRes.id,
77
+ date: new Date().toISOString(),
78
+ model: infoRes.modelId,
79
+ },
80
+ }, { suppressWorkflows: true });
81
+ return { status: "completed" };
82
+ }
83
+ //# sourceMappingURL=generateDocumentProperties.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"generateDocumentProperties.js","sourceRoot":"","sources":["../../../src/activities/generateDocumentProperties.ts"],"names":[],"mappings":";;AAsBA,gEAiGC;AAvHD,mDAA2C;AAE3C,wEAAgE;AAChE,kDAAoE;AACpE,mEAAqG;AAErG,MAAM,uBAAuB,GAAG,wBAAwB,CAAC;AAgBlD,KAAK,UAAU,0BAA0B,CAC5C,OAAsE;IAEtE,MAAM,OAAO,GAAG,MAAM,IAAA,kCAAa,EAAmC,OAAO,CAAC,CAAC;IAC/E,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;IAC7C,MAAM,eAAe,GAAG,MAAM,CAAC,eAAe,IAAI,uBAAuB,CAAC;IAE1E,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,YAAY,EAAE,CAAC;IAE7C,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC7D,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAE7E,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,CAAC,UAAU,IAAI,CAAC,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QAChF,cAAG,CAAC,IAAI,CAAC,UAAU,QAAQ,6BAA6B,CAAC,CAAC;QAC1D,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;IAClD,CAAC;IAED,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;QAC/B,cAAG,CAAC,IAAI,CAAC,UAAU,QAAQ,gBAAgB,CAAC,CAAC;QAC7C,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,2BAA2B,EAAE,CAAC;IAC3F,CAAC;IAED,MAAM,WAAW,GAAG,GAAG,EAAE;QACrB,IAAI,GAAG,CAAC,OAAO,EAAE,IAAI,EAAE,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC1C,OAAO,QAAQ,GAAG,GAAG,CAAC,EAAE,CAAC;QAC7B,CAAC;QAED,IAAI,MAAM,CAAC,UAAU,IAAI,GAAG,CAAC,OAAO,EAAE,IAAI,EAAE,UAAU,CAAC,iBAAiB,CAAC,EAAE,CAAC;YACxE,OAAO,QAAQ,GAAG,GAAG,CAAC,EAAE,CAAC;QAC7B,CAAC;QAED,cAAG,CAAC,IAAI,CAAC,UAAU,QAAQ,yBAAyB,CAAC,CAAC;QACtD,OAAO,SAAS,CAAC;IACrB,CAAC,CAAC;IAEF,MAAM,OAAO,GAAG,GAAG,CAAC,IAAI;QACpB,CAAC,CAAC,IAAA,4BAAgB,EAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,QAAQ,IAAI,KAAK,CAAC;QACtD,CAAC,CAAC,SAAS,CAAC;IAEhB,MAAM,UAAU,GAAG;QACf,OAAO,EAAE,OAAO;QAChB,KAAK,EAAE,WAAW,EAAE,IAAI,SAAS;QACjC,aAAa,EAAE,OAAO,EAAE,aAAa,EAAE,aAAa,IAAI,SAAS;KACpE,CAAC;IAEF,cAAG,CAAC,IAAI,CACJ,uCAAuC,QAAQ,cAAc,IAAI,CAAC,IAAI,EAAE,EACxE,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,SAAS,CAC9C,CAAC;IAEF,MAAM,OAAO,GAAG,MAAM,IAAA,sDAA8B,EAChD,MAAM,EACN,eAAe,EACf;QACI,GAAG,MAAM;QACT,sBAAsB,EAAE,IAAI;QAC5B,aAAa,EAAE,IAAI,CAAC,aAAa;QACjC,eAAe,EAAE,IAAI,CAAC,WAAW;KACpC,EACD,UAAU,EACV,OAAO,CAAC,UAAU,IAAI,KAAK,CAC9B,CAAC;IAEF,MAAM,OAAO,GAAG,GAAG,EAAE;QACjB,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;YACX,OAAO,SAAS,CAAC;QACrB,CAAC;QACD,IAAI,IAAI,GAAG,EAAE,CAAC;QACd,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QAC3C,IAAI,UAAU,CAAC,KAAK,EAAE,CAAC;YACnB,IAAI,IAAI,UAAU,CAAC,KAAK,GAAG,IAAI,CAAC;QACpC,CAAC;QACD,IAAI,UAAU,CAAC,WAAW,EAAE,CAAC;YACzB,IAAI,IAAI,UAAU,CAAC,WAAW,CAAC;QACnC,CAAC;QACD,IAAI,IAAI,EAAE,CAAC;YACP,OAAO,IAAI,CAAC;QAChB,CAAC;aAAM,CAAC;YACJ,OAAO,SAAS,CAAC;QACrB,CAAC;IACL,CAAC,CAAC;IAEF,cAAG,CAAC,IAAI,CAAC,qCAAqC,QAAQ,cAAc,IAAI,CAAC,IAAI,EAAE,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC;IACxG,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE;QAChC,UAAU,EAAE;YACR,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE;YAC1B,IAAI,EAAE,GAAG,CAAC,SAAS;SACtB;QACD,IAAI,EAAE,OAAO,EAAE;QACf,mBAAmB,EAAE;YACjB,EAAE,EAAE,OAAO,CAAC,EAAE;YACd,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YAC9B,KAAK,EAAE,OAAO,CAAC,OAAO;SACzB;KACJ,EAAE,EAAE,iBAAiB,EAAE,IAAI,EAAE,CAAC,CAAC;IAEhC,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC;AACnC,CAAC"}