@vertesia/workflow 1.0.0-dev.20260203.130115Z → 1.0.0-dev.20260225.024852Z
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/cjs/activities/advanced/createDocumentTypeFromInteractionRun.js +33 -0
- package/lib/cjs/activities/advanced/createDocumentTypeFromInteractionRun.js.map +1 -0
- package/lib/cjs/activities/advanced/createOrUpdateDocumentFromInteractionRun.js +73 -0
- package/lib/cjs/activities/advanced/createOrUpdateDocumentFromInteractionRun.js.map +1 -0
- package/lib/cjs/activities/advanced/updateDocumentFromInteractionRun.js +19 -0
- package/lib/cjs/activities/advanced/updateDocumentFromInteractionRun.js.map +1 -0
- package/lib/cjs/activities/chunkDocument.js +87 -0
- package/lib/cjs/activities/chunkDocument.js.map +1 -0
- package/lib/cjs/activities/copyParentArtifacts.js +127 -0
- package/lib/cjs/activities/copyParentArtifacts.js.map +1 -0
- package/lib/cjs/activities/createDocumentFromOther.js +64 -0
- package/lib/cjs/activities/createDocumentFromOther.js.map +1 -0
- package/lib/cjs/activities/executeInteraction.js +194 -0
- package/lib/cjs/activities/executeInteraction.js.map +1 -0
- package/lib/cjs/activities/extractDocumentText.js +196 -0
- package/lib/cjs/activities/extractDocumentText.js.map +1 -0
- package/lib/cjs/activities/generateDocumentProperties.js +85 -0
- package/lib/cjs/activities/generateDocumentProperties.js.map +1 -0
- package/lib/cjs/activities/generateEmbeddings.js +267 -0
- package/lib/cjs/activities/generateEmbeddings.js.map +1 -0
- package/lib/cjs/activities/generateOrAssignContentType.js +125 -0
- package/lib/cjs/activities/generateOrAssignContentType.js.map +1 -0
- package/lib/cjs/activities/getObjectFromStore.js +20 -0
- package/lib/cjs/activities/getObjectFromStore.js.map +1 -0
- package/lib/cjs/activities/handleError.js +22 -0
- package/lib/cjs/activities/handleError.js.map +1 -0
- package/lib/cjs/activities/index-dsl.js +53 -0
- package/lib/cjs/activities/index-dsl.js.map +1 -0
- package/lib/cjs/activities/index.js +21 -0
- package/lib/cjs/activities/index.js.map +1 -0
- package/lib/cjs/activities/media/prepareAudio.js +239 -0
- package/lib/cjs/activities/media/prepareAudio.js.map +1 -0
- package/lib/cjs/activities/media/prepareVideo.js +429 -0
- package/lib/cjs/activities/media/prepareVideo.js.map +1 -0
- package/lib/cjs/activities/media/processPdfWithTextract.js +103 -0
- package/lib/cjs/activities/media/processPdfWithTextract.js.map +1 -0
- package/lib/cjs/activities/media/saveGladiaTranscription.js +95 -0
- package/lib/cjs/activities/media/saveGladiaTranscription.js.map +1 -0
- package/lib/cjs/activities/media/transcribeMediaWithGladia.js +99 -0
- package/lib/cjs/activities/media/transcribeMediaWithGladia.js.map +1 -0
- package/lib/cjs/activities/mergeChildArtifacts.js +150 -0
- package/lib/cjs/activities/mergeChildArtifacts.js.map +1 -0
- package/lib/cjs/activities/notifyWebhook.js +167 -0
- package/lib/cjs/activities/notifyWebhook.js.map +1 -0
- package/lib/cjs/activities/rateLimiter.js +30 -0
- package/lib/cjs/activities/rateLimiter.js.map +1 -0
- package/lib/cjs/activities/renditions/generateImageRendition.js +66 -0
- package/lib/cjs/activities/renditions/generateImageRendition.js.map +1 -0
- package/lib/cjs/activities/renditions/generateVideoRendition.js +200 -0
- package/lib/cjs/activities/renditions/generateVideoRendition.js.map +1 -0
- package/lib/cjs/activities/setDocumentStatus.js +26 -0
- package/lib/cjs/activities/setDocumentStatus.js.map +1 -0
- package/lib/cjs/conversion/TextractProcessor.js +417 -0
- package/lib/cjs/conversion/TextractProcessor.js.map +1 -0
- package/lib/cjs/conversion/image.js +149 -0
- package/lib/cjs/conversion/image.js.map +1 -0
- package/lib/cjs/conversion/markitdown.js +42 -0
- package/lib/cjs/conversion/markitdown.js.map +1 -0
- package/lib/cjs/conversion/mutool.js +147 -0
- package/lib/cjs/conversion/mutool.js.map +1 -0
- package/lib/cjs/conversion/pandoc.js +39 -0
- package/lib/cjs/conversion/pandoc.js.map +1 -0
- package/lib/cjs/dsl/conditions.js +81 -0
- package/lib/cjs/dsl/conditions.js.map +1 -0
- package/lib/cjs/dsl/dsl-workflow.js +380 -0
- package/lib/cjs/dsl/dsl-workflow.js.map +1 -0
- package/lib/cjs/dsl/dslProxyActivities.js +23 -0
- package/lib/cjs/dsl/dslProxyActivities.js.map +1 -0
- package/lib/cjs/dsl/projections.js +59 -0
- package/lib/cjs/dsl/projections.js.map +1 -0
- package/lib/cjs/dsl/setup/ActivityContext.js +178 -0
- package/lib/cjs/dsl/setup/ActivityContext.js.map +1 -0
- package/lib/cjs/dsl/setup/fetch/DataProvider.js +51 -0
- package/lib/cjs/dsl/setup/fetch/DataProvider.js.map +1 -0
- package/lib/cjs/dsl/setup/fetch/index.js +16 -0
- package/lib/cjs/dsl/setup/fetch/index.js.map +1 -0
- package/lib/cjs/dsl/setup/fetch/providers.js +67 -0
- package/lib/cjs/dsl/setup/fetch/providers.js.map +1 -0
- package/lib/cjs/dsl/test/test-child-workflow.js +10 -0
- package/lib/cjs/dsl/test/test-child-workflow.js.map +1 -0
- package/lib/cjs/dsl/validation.js +122 -0
- package/lib/cjs/dsl/validation.js.map +1 -0
- package/lib/cjs/dsl/vars.js +341 -0
- package/lib/cjs/dsl/vars.js.map +1 -0
- package/lib/cjs/dsl/walk.js +100 -0
- package/lib/cjs/dsl/walk.js.map +1 -0
- package/lib/cjs/dsl.js +20 -0
- package/lib/cjs/dsl.js.map +1 -0
- package/lib/cjs/errors.js +89 -0
- package/lib/cjs/errors.js.map +1 -0
- package/lib/cjs/index.js +57 -0
- package/lib/cjs/index.js.map +1 -0
- package/lib/cjs/iterative-generation/activities/extractToc.js +47 -0
- package/lib/cjs/iterative-generation/activities/extractToc.js.map +1 -0
- package/lib/cjs/iterative-generation/activities/finalizeOutput.js +72 -0
- package/lib/cjs/iterative-generation/activities/finalizeOutput.js.map +1 -0
- package/lib/cjs/iterative-generation/activities/generatePart.js +78 -0
- package/lib/cjs/iterative-generation/activities/generatePart.js.map +1 -0
- package/lib/cjs/iterative-generation/activities/generateToc.js +86 -0
- package/lib/cjs/iterative-generation/activities/generateToc.js.map +1 -0
- package/lib/cjs/iterative-generation/activities/index.js +12 -0
- package/lib/cjs/iterative-generation/activities/index.js.map +1 -0
- package/lib/cjs/iterative-generation/iterativeGenerationWorkflow.js +56 -0
- package/lib/cjs/iterative-generation/iterativeGenerationWorkflow.js.map +1 -0
- package/lib/cjs/iterative-generation/types.js +5 -0
- package/lib/cjs/iterative-generation/types.js.map +1 -0
- package/lib/cjs/iterative-generation/utils.js +121 -0
- package/lib/cjs/iterative-generation/utils.js.map +1 -0
- package/lib/cjs/package.json +3 -0
- package/lib/cjs/result-types.js +10 -0
- package/lib/cjs/result-types.js.map +1 -0
- package/lib/cjs/system/notifyWebhookWorkflow.js +53 -0
- package/lib/cjs/system/notifyWebhookWorkflow.js.map +1 -0
- package/lib/cjs/system/recalculateEmbeddingsWorkflow.js +33 -0
- package/lib/cjs/system/recalculateEmbeddingsWorkflow.js.map +1 -0
- package/lib/cjs/utils/auth.js +15 -0
- package/lib/cjs/utils/auth.js.map +1 -0
- package/lib/cjs/utils/blobs.js +64 -0
- package/lib/cjs/utils/blobs.js.map +1 -0
- package/lib/cjs/utils/chunks.js +14 -0
- package/lib/cjs/utils/chunks.js.map +1 -0
- package/lib/cjs/utils/client.js +31 -0
- package/lib/cjs/utils/client.js.map +1 -0
- package/lib/cjs/utils/expand-vars.js +33 -0
- package/lib/cjs/utils/expand-vars.js.map +1 -0
- package/lib/cjs/utils/memory.js +65 -0
- package/lib/cjs/utils/memory.js.map +1 -0
- package/lib/cjs/utils/renditions.js +92 -0
- package/lib/cjs/utils/renditions.js.map +1 -0
- package/lib/cjs/utils/storage.js +54 -0
- package/lib/cjs/utils/storage.js.map +1 -0
- package/lib/cjs/utils/text-preview-utils.js +43 -0
- package/lib/cjs/utils/text-preview-utils.js.map +1 -0
- package/lib/cjs/utils/tokens.js +38 -0
- package/lib/cjs/utils/tokens.js.map +1 -0
- package/lib/cjs/vars.js +20 -0
- package/lib/cjs/vars.js.map +1 -0
- package/lib/cjs/workflows.js +15 -0
- package/lib/cjs/workflows.js.map +1 -0
- package/lib/esm/activities/advanced/createDocumentTypeFromInteractionRun.js +30 -0
- package/lib/esm/activities/advanced/createDocumentTypeFromInteractionRun.js.map +1 -0
- package/lib/esm/activities/advanced/createOrUpdateDocumentFromInteractionRun.js +70 -0
- package/lib/esm/activities/advanced/createOrUpdateDocumentFromInteractionRun.js.map +1 -0
- package/lib/esm/activities/advanced/updateDocumentFromInteractionRun.js +16 -0
- package/lib/esm/activities/advanced/updateDocumentFromInteractionRun.js.map +1 -0
- package/lib/esm/activities/chunkDocument.js +84 -0
- package/lib/esm/activities/chunkDocument.js.map +1 -0
- package/lib/esm/activities/copyParentArtifacts.js +124 -0
- package/lib/esm/activities/copyParentArtifacts.js.map +1 -0
- package/lib/esm/activities/createDocumentFromOther.js +58 -0
- package/lib/esm/activities/createDocumentFromOther.js.map +1 -0
- package/lib/esm/activities/executeInteraction.js +190 -0
- package/lib/esm/activities/executeInteraction.js.map +1 -0
- package/lib/esm/activities/extractDocumentText.js +193 -0
- package/lib/esm/activities/extractDocumentText.js.map +1 -0
- package/lib/esm/activities/generateDocumentProperties.js +82 -0
- package/lib/esm/activities/generateDocumentProperties.js.map +1 -0
- package/lib/esm/activities/generateEmbeddings.js +264 -0
- package/lib/esm/activities/generateEmbeddings.js.map +1 -0
- package/lib/esm/activities/generateOrAssignContentType.js +122 -0
- package/lib/esm/activities/generateOrAssignContentType.js.map +1 -0
- package/lib/esm/activities/getObjectFromStore.js +17 -0
- package/lib/esm/activities/getObjectFromStore.js.map +1 -0
- package/lib/esm/activities/handleError.js +19 -0
- package/lib/esm/activities/handleError.js.map +1 -0
- package/lib/esm/activities/index-dsl.js +26 -0
- package/lib/esm/activities/index-dsl.js.map +1 -0
- package/lib/esm/activities/index.js +5 -0
- package/lib/esm/activities/index.js.map +1 -0
- package/lib/esm/activities/media/prepareAudio.js +200 -0
- package/lib/esm/activities/media/prepareAudio.js.map +1 -0
- package/lib/esm/activities/media/prepareVideo.js +390 -0
- package/lib/esm/activities/media/prepareVideo.js.map +1 -0
- package/lib/esm/activities/media/processPdfWithTextract.js +99 -0
- package/lib/esm/activities/media/processPdfWithTextract.js.map +1 -0
- package/lib/esm/activities/media/saveGladiaTranscription.js +92 -0
- package/lib/esm/activities/media/saveGladiaTranscription.js.map +1 -0
- package/lib/esm/activities/media/transcribeMediaWithGladia.js +96 -0
- package/lib/esm/activities/media/transcribeMediaWithGladia.js.map +1 -0
- package/lib/esm/activities/mergeChildArtifacts.js +147 -0
- package/lib/esm/activities/mergeChildArtifacts.js.map +1 -0
- package/lib/esm/activities/notifyWebhook.js +164 -0
- package/lib/esm/activities/notifyWebhook.js.map +1 -0
- package/lib/esm/activities/rateLimiter.js +27 -0
- package/lib/esm/activities/rateLimiter.js.map +1 -0
- package/lib/esm/activities/renditions/generateImageRendition.js +63 -0
- package/lib/esm/activities/renditions/generateImageRendition.js.map +1 -0
- package/lib/esm/activities/renditions/generateVideoRendition.js +194 -0
- package/lib/esm/activities/renditions/generateVideoRendition.js.map +1 -0
- package/lib/esm/activities/setDocumentStatus.js +23 -0
- package/lib/esm/activities/setDocumentStatus.js.map +1 -0
- package/lib/esm/conversion/TextractProcessor.js +410 -0
- package/lib/esm/conversion/TextractProcessor.js.map +1 -0
- package/lib/esm/conversion/image.js +143 -0
- package/lib/esm/conversion/image.js.map +1 -0
- package/lib/esm/conversion/markitdown.js +36 -0
- package/lib/esm/conversion/markitdown.js.map +1 -0
- package/lib/esm/conversion/mutool.js +139 -0
- package/lib/esm/conversion/mutool.js.map +1 -0
- package/lib/esm/conversion/pandoc.js +36 -0
- package/lib/esm/conversion/pandoc.js.map +1 -0
- package/lib/esm/dsl/conditions.js +75 -0
- package/lib/esm/dsl/conditions.js.map +1 -0
- package/lib/esm/dsl/dsl-workflow.js +373 -0
- package/lib/esm/dsl/dsl-workflow.js.map +1 -0
- package/lib/esm/dsl/dslProxyActivities.js +20 -0
- package/lib/esm/dsl/dslProxyActivities.js.map +1 -0
- package/lib/esm/dsl/projections.js +55 -0
- package/lib/esm/dsl/projections.js.map +1 -0
- package/lib/esm/dsl/setup/ActivityContext.js +173 -0
- package/lib/esm/dsl/setup/ActivityContext.js.map +1 -0
- package/lib/esm/dsl/setup/fetch/DataProvider.js +47 -0
- package/lib/esm/dsl/setup/fetch/DataProvider.js.map +1 -0
- package/lib/esm/dsl/setup/fetch/index.js +12 -0
- package/lib/esm/dsl/setup/fetch/index.js.map +1 -0
- package/lib/esm/dsl/setup/fetch/providers.js +61 -0
- package/lib/esm/dsl/setup/fetch/providers.js.map +1 -0
- package/lib/esm/dsl/test/test-child-workflow.js +5 -0
- package/lib/esm/dsl/test/test-child-workflow.js.map +1 -0
- package/lib/esm/dsl/validation.js +118 -0
- package/lib/esm/dsl/validation.js.map +1 -0
- package/lib/esm/dsl/vars.js +335 -0
- package/lib/esm/dsl/vars.js.map +1 -0
- package/lib/esm/dsl/walk.js +96 -0
- package/lib/esm/dsl/walk.js.map +1 -0
- package/lib/esm/dsl.js +4 -0
- package/lib/esm/dsl.js.map +1 -0
- package/lib/esm/errors.js +78 -0
- package/lib/esm/errors.js.map +1 -0
- package/lib/esm/index.js +39 -0
- package/lib/esm/index.js.map +1 -0
- package/lib/esm/iterative-generation/activities/extractToc.js +44 -0
- package/lib/esm/iterative-generation/activities/extractToc.js.map +1 -0
- package/lib/esm/iterative-generation/activities/finalizeOutput.js +69 -0
- package/lib/esm/iterative-generation/activities/finalizeOutput.js.map +1 -0
- package/lib/esm/iterative-generation/activities/generatePart.js +75 -0
- package/lib/esm/iterative-generation/activities/generatePart.js.map +1 -0
- package/lib/esm/iterative-generation/activities/generateToc.js +83 -0
- package/lib/esm/iterative-generation/activities/generateToc.js.map +1 -0
- package/lib/esm/iterative-generation/activities/index.js +5 -0
- package/lib/esm/iterative-generation/activities/index.js.map +1 -0
- package/lib/esm/iterative-generation/iterativeGenerationWorkflow.js +53 -0
- package/lib/esm/iterative-generation/iterativeGenerationWorkflow.js.map +1 -0
- package/lib/esm/iterative-generation/types.js +2 -0
- package/lib/esm/iterative-generation/types.js.map +1 -0
- package/lib/esm/iterative-generation/utils.js +112 -0
- package/lib/esm/iterative-generation/utils.js.map +1 -0
- package/lib/esm/result-types.js +7 -0
- package/lib/esm/result-types.js.map +1 -0
- package/lib/esm/system/notifyWebhookWorkflow.js +50 -0
- package/lib/esm/system/notifyWebhookWorkflow.js.map +1 -0
- package/lib/esm/system/recalculateEmbeddingsWorkflow.js +30 -0
- package/lib/esm/system/recalculateEmbeddingsWorkflow.js.map +1 -0
- package/lib/esm/utils/auth.js +8 -0
- package/lib/esm/utils/auth.js.map +1 -0
- package/lib/esm/utils/blobs.js +54 -0
- package/lib/esm/utils/blobs.js.map +1 -0
- package/lib/esm/utils/chunks.js +9 -0
- package/lib/esm/utils/chunks.js.map +1 -0
- package/lib/esm/utils/client.js +27 -0
- package/lib/esm/utils/client.js.map +1 -0
- package/lib/esm/utils/expand-vars.js +30 -0
- package/lib/esm/utils/expand-vars.js.map +1 -0
- package/lib/esm/utils/memory.js +55 -0
- package/lib/esm/utils/memory.js.map +1 -0
- package/lib/esm/utils/renditions.js +84 -0
- package/lib/esm/utils/renditions.js.map +1 -0
- package/lib/esm/utils/storage.js +45 -0
- package/lib/esm/utils/storage.js.map +1 -0
- package/lib/esm/utils/text-preview-utils.js +38 -0
- package/lib/esm/utils/text-preview-utils.js.map +1 -0
- package/lib/esm/utils/tokens.js +34 -0
- package/lib/esm/utils/tokens.js.map +1 -0
- package/lib/esm/vars.js +4 -0
- package/lib/esm/vars.js.map +1 -0
- package/lib/esm/workflows.js +8 -0
- package/lib/esm/workflows.js.map +1 -0
- package/lib/tsconfig.tsbuildinfo +1 -0
- package/lib/types/activities/advanced/createDocumentTypeFromInteractionRun.d.ts +17 -0
- package/lib/types/activities/advanced/createDocumentTypeFromInteractionRun.d.ts.map +1 -0
- package/lib/types/activities/advanced/createOrUpdateDocumentFromInteractionRun.d.ts +39 -0
- package/lib/types/activities/advanced/createOrUpdateDocumentFromInteractionRun.d.ts.map +1 -0
- package/lib/types/activities/advanced/updateDocumentFromInteractionRun.d.ts +19 -0
- package/lib/types/activities/advanced/updateDocumentFromInteractionRun.d.ts.map +1 -0
- package/lib/types/activities/chunkDocument.d.ts +33 -0
- package/lib/types/activities/chunkDocument.d.ts.map +1 -0
- package/lib/types/activities/copyParentArtifacts.d.ts +19 -0
- package/lib/types/activities/copyParentArtifacts.d.ts.map +1 -0
- package/lib/types/activities/createDocumentFromOther.d.ts +21 -0
- package/lib/types/activities/createDocumentFromOther.d.ts.map +1 -0
- package/lib/types/activities/executeInteraction.d.ts +61 -0
- package/lib/types/activities/executeInteraction.d.ts.map +1 -0
- package/lib/types/activities/extractDocumentText.d.ts +11 -0
- package/lib/types/activities/extractDocumentText.d.ts.map +1 -0
- package/lib/types/activities/generateDocumentProperties.d.ts +32 -0
- package/lib/types/activities/generateDocumentProperties.d.ts.map +1 -0
- package/lib/types/activities/generateEmbeddings.d.ts +53 -0
- package/lib/types/activities/generateEmbeddings.d.ts.map +1 -0
- package/lib/types/activities/generateOrAssignContentType.d.ts +44 -0
- package/lib/types/activities/generateOrAssignContentType.d.ts.map +1 -0
- package/lib/types/activities/getObjectFromStore.d.ts +14 -0
- package/lib/types/activities/getObjectFromStore.d.ts.map +1 -0
- package/lib/types/activities/handleError.d.ts +6 -0
- package/lib/types/activities/handleError.d.ts.map +1 -0
- package/lib/types/activities/index-dsl.d.ts +26 -0
- package/lib/types/activities/index-dsl.d.ts.map +1 -0
- package/lib/types/activities/index.d.ts +5 -0
- package/lib/types/activities/index.d.ts.map +1 -0
- package/lib/types/activities/media/prepareAudio.d.ts +25 -0
- package/lib/types/activities/media/prepareAudio.d.ts.map +1 -0
- package/lib/types/activities/media/prepareVideo.d.ts +30 -0
- package/lib/types/activities/media/prepareVideo.d.ts.map +1 -0
- package/lib/types/activities/media/processPdfWithTextract.d.ts +26 -0
- package/lib/types/activities/media/processPdfWithTextract.d.ts.map +1 -0
- package/lib/types/activities/media/saveGladiaTranscription.d.ts +15 -0
- package/lib/types/activities/media/saveGladiaTranscription.d.ts.map +1 -0
- package/lib/types/activities/media/transcribeMediaWithGladia.d.ts +20 -0
- package/lib/types/activities/media/transcribeMediaWithGladia.d.ts.map +1 -0
- package/lib/types/activities/mergeChildArtifacts.d.ts +24 -0
- package/lib/types/activities/mergeChildArtifacts.d.ts.map +1 -0
- package/lib/types/activities/notifyWebhook.d.ts +27 -0
- package/lib/types/activities/notifyWebhook.d.ts.map +1 -0
- package/lib/types/activities/rateLimiter.d.ts +11 -0
- package/lib/types/activities/rateLimiter.d.ts.map +1 -0
- package/lib/types/activities/renditions/generateImageRendition.d.ts +14 -0
- package/lib/types/activities/renditions/generateImageRendition.d.ts.map +1 -0
- package/lib/types/activities/renditions/generateVideoRendition.d.ts +15 -0
- package/lib/types/activities/renditions/generateVideoRendition.d.ts.map +1 -0
- package/lib/types/activities/setDocumentStatus.d.ts +15 -0
- package/lib/types/activities/setDocumentStatus.d.ts.map +1 -0
- package/lib/types/conversion/TextractProcessor.d.ts +45 -0
- package/lib/types/conversion/TextractProcessor.d.ts.map +1 -0
- package/lib/types/conversion/image.d.ts +13 -0
- package/lib/types/conversion/image.d.ts.map +1 -0
- package/lib/types/conversion/markitdown.d.ts +2 -0
- package/lib/types/conversion/markitdown.d.ts.map +1 -0
- package/lib/types/conversion/mutool.d.ts +19 -0
- package/lib/types/conversion/mutool.d.ts.map +1 -0
- package/lib/types/conversion/pandoc.d.ts +2 -0
- package/lib/types/conversion/pandoc.d.ts.map +1 -0
- package/lib/types/dsl/conditions.d.ts +2 -0
- package/lib/types/dsl/conditions.d.ts.map +1 -0
- package/lib/types/dsl/dsl-workflow.d.ts +5 -0
- package/lib/types/dsl/dsl-workflow.d.ts.map +1 -0
- package/lib/types/dsl/dslProxyActivities.d.ts +10 -0
- package/lib/types/dsl/dslProxyActivities.d.ts.map +1 -0
- package/lib/types/dsl/projections.d.ts +4 -0
- package/lib/types/dsl/projections.d.ts.map +1 -0
- package/lib/types/dsl/setup/ActivityContext.d.ts +47 -0
- package/lib/types/dsl/setup/ActivityContext.d.ts.map +1 -0
- package/lib/types/dsl/setup/fetch/DataProvider.d.ts +9 -0
- package/lib/types/dsl/setup/fetch/DataProvider.d.ts.map +1 -0
- package/lib/types/dsl/setup/fetch/index.d.ts +6 -0
- package/lib/types/dsl/setup/fetch/index.d.ts.map +1 -0
- package/lib/types/dsl/setup/fetch/providers.d.ts +25 -0
- package/lib/types/dsl/setup/fetch/providers.d.ts.map +1 -0
- package/lib/types/dsl/test/test-child-workflow.d.ts +4 -0
- package/lib/types/dsl/test/test-child-workflow.d.ts.map +1 -0
- package/lib/types/dsl/validation.d.ts +4 -0
- package/lib/types/dsl/validation.d.ts.map +1 -0
- package/lib/types/dsl/vars.d.ts +48 -0
- package/lib/types/dsl/vars.d.ts.map +1 -0
- package/lib/types/dsl/walk.d.ts +18 -0
- package/lib/types/dsl/walk.d.ts.map +1 -0
- package/lib/types/dsl.d.ts +4 -0
- package/lib/types/dsl.d.ts.map +1 -0
- package/lib/types/errors.d.ts +41 -0
- package/lib/types/errors.d.ts.map +1 -0
- package/lib/types/index.d.ts +38 -0
- package/lib/types/index.d.ts.map +1 -0
- package/lib/types/iterative-generation/activities/extractToc.d.ts +10 -0
- package/lib/types/iterative-generation/activities/extractToc.d.ts.map +1 -0
- package/lib/types/iterative-generation/activities/finalizeOutput.d.ts +3 -0
- package/lib/types/iterative-generation/activities/finalizeOutput.d.ts.map +1 -0
- package/lib/types/iterative-generation/activities/generatePart.d.ts +3 -0
- package/lib/types/iterative-generation/activities/generatePart.d.ts.map +1 -0
- package/lib/types/iterative-generation/activities/generateToc.d.ts +4 -0
- package/lib/types/iterative-generation/activities/generateToc.d.ts.map +1 -0
- package/lib/types/iterative-generation/activities/index.d.ts +5 -0
- package/lib/types/iterative-generation/activities/index.d.ts.map +1 -0
- package/lib/types/iterative-generation/iterativeGenerationWorkflow.d.ts +3 -0
- package/lib/types/iterative-generation/iterativeGenerationWorkflow.d.ts.map +1 -0
- package/lib/types/iterative-generation/types.d.ts +79 -0
- package/lib/types/iterative-generation/types.d.ts.map +1 -0
- package/lib/types/iterative-generation/utils.d.ts +26 -0
- package/lib/types/iterative-generation/utils.d.ts.map +1 -0
- package/lib/types/result-types.d.ts +26 -0
- package/lib/types/result-types.d.ts.map +1 -0
- package/lib/types/system/notifyWebhookWorkflow.d.ts +8 -0
- package/lib/types/system/notifyWebhookWorkflow.d.ts.map +1 -0
- package/lib/types/system/recalculateEmbeddingsWorkflow.d.ts +25 -0
- package/lib/types/system/recalculateEmbeddingsWorkflow.d.ts.map +1 -0
- package/lib/types/utils/auth.d.ts +4 -0
- package/lib/types/utils/auth.d.ts.map +1 -0
- package/lib/types/utils/blobs.d.ts +7 -0
- package/lib/types/utils/blobs.d.ts.map +1 -0
- package/lib/types/utils/chunks.d.ts +9 -0
- package/lib/types/utils/chunks.d.ts.map +1 -0
- package/lib/types/utils/client.d.ts +8 -0
- package/lib/types/utils/client.d.ts.map +1 -0
- package/lib/types/utils/expand-vars.d.ts +8 -0
- package/lib/types/utils/expand-vars.d.ts.map +1 -0
- package/lib/types/utils/memory.d.ts +8 -0
- package/lib/types/utils/memory.d.ts.map +1 -0
- package/lib/types/utils/renditions.d.ts +25 -0
- package/lib/types/utils/renditions.d.ts.map +1 -0
- package/lib/types/utils/storage.d.ts +16 -0
- package/lib/types/utils/storage.d.ts.map +1 -0
- package/lib/types/utils/text-preview-utils.d.ts +15 -0
- package/lib/types/utils/text-preview-utils.d.ts.map +1 -0
- package/lib/types/utils/tokens.d.ts +11 -0
- package/lib/types/utils/tokens.d.ts.map +1 -0
- package/lib/types/vars.d.ts +3 -0
- package/lib/types/vars.d.ts.map +1 -0
- package/lib/types/workflows.d.ts +8 -0
- package/lib/types/workflows.d.ts.map +1 -0
- package/lib/workflows-bundle.js +17846 -0
- package/package.json +6 -7
- package/src/activities/advanced/createOrUpdateDocumentFromInteractionRun.ts +1 -1
- package/src/activities/chunkDocument.ts +3 -1
- package/src/activities/extractDocumentText.ts +84 -28
- package/src/activities/generateDocumentProperties.ts +4 -2
- package/src/activities/generateEmbeddings.ts +9 -9
- package/src/activities/generateOrAssignContentType.ts +2 -2
- package/src/activities/index-dsl.ts +4 -3
- package/src/activities/media/saveGladiaTranscription.test.ts +406 -0
- package/src/activities/media/saveGladiaTranscription.ts +41 -26
- package/src/activities/media/transcribeMediaWithGladia.test.ts +583 -0
- package/src/activities/media/transcribeMediaWithGladia.ts +46 -25
- package/src/activities/notifyWebhook.test.ts +121 -8
- package/src/activities/notifyWebhook.ts +10 -2
- package/src/activities/renditions/generateImageRendition.ts +2 -2
- package/src/conversion/image.test.ts +1 -0
- package/src/conversion/image.ts +10 -10
- package/src/dsl/dsl-workflow.ts +49 -9
- package/src/dsl/setup/ActivityContext.ts +73 -0
- package/src/dsl.ts +1 -0
- package/src/errors.ts +14 -0
- package/src/index.ts +6 -5
- package/src/result-types.ts +5 -1
- package/src/utils/renditions.ts +11 -5
- package/src/utils/text-preview-utils.ts +62 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"executeInteraction.js","sourceRoot":"","sources":["../../../src/activities/executeInteraction.ts"],"names":[],"mappings":";;AAiIA,gDAqFC;AAED,wEAmFC;AA1SD,mDAAyD;AAEzD,gDAAyD;AACzD,6CAQ0B;AAC1B,0DAAsD;AACtD,wEAAgE;AAChE,4CAA6G;AAC7G,kDAAoE;AACpE,mCAAkC;AAElC,UAAU;AACV,YAAY;AACZ,MAAM,IAAI,GAAoB;IAC1B,IAAI,EAAE,oBAAoB;IAC1B,MAAM,EAAE,CAAC,cAAc,EAAE,YAAY,EAAE,WAAW,CAAC;IACnD,MAAM,EAAE;QACJ,YAAY,EAAE,UAAU;QACxB,eAAe,EAAE,iBAAiB;QAClC,KAAK,EAAE,2BAA2B;QAClC,WAAW,EAAE,OAAO;QACpB,UAAU,EAAE,GAAG;QACf,WAAW,EAAE,GAAG;QAChB,IAAI,EAAE,CAAC,MAAM,CAAC;QACd,aAAa,EAAE,0BAA0B;QACzC,WAAW,EAAE;YACT,SAAS,EAAE,cAAc;YACzB,QAAQ,EAAE,kBAAkB;SAC/B;KACJ;IACD,KAAK,EAAE;QACH,SAAS,EAAE;YACP,IAAI,EAAE,UAAU;YAChB,KAAK,EAAE;gBACH,EAAE,EAAE,EAAE,GAAG,EAAE,cAAc,EAAE;aAC9B;YACD,MAAM,EAAE,OAAO;SAClB;QACD,QAAQ,EAAE;YACN,IAAI,EAAE,UAAU;YAChB,KAAK,EAAE,CAAC;YACR,KAAK,EAAE;gBACH,EAAE,EAAE,eAAe;aACtB;YACD,MAAM,EAAE,OAAO;YACf,YAAY,EAAE,OAAO;SACxB;QACD,OAAO,EAAE;YACL,IAAI,EAAE,eAAe;YACrB,KAAK,EAAE,CAAC;YACR,KAAK,EAAE;gBACH,EAAE,EAAE,cAAc;aACrB;YACD,MAAM,EAAE,gBAAgB;SAC3B;KACJ;CACJ,CAAC;AAiEK,KAAK,UAAU,kBAAkB,CAAC,OAA8D;IACnG,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,IAAA,kCAAa,EAA2B,OAAO,CAAC,CAAC;IAElF,MAAM,EAAE,eAAe,EAAE,WAAW,EAAE,kBAAkB,EAAE,cAAc,EAAE,GAAG,MAAM,CAAC;IACpF,IAAI,cAAc,EAAE,CAAC;QACjB,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;IAC/C,CAAC;IAED,IAAI,CAAC,eAAe,EAAE,CAAC;QACnB,cAAG,CAAC,KAAK,CAAC,yBAAyB,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;QACjD,MAAM,IAAI,sCAA0B,CAAC,iBAAiB,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC9E,CAAC;IAED,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;QAClB,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;QACjC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;YAClD,WAAW,CAAC,GAAG,CAAC,GAAG,IAAA,4BAAgB,EAAC,WAAW,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,CAAC;QACjE,CAAC;IACL,CAAC;IAED,IAAI,CAAC;QACD,MAAM,GAAG,GAAG,MAAM,8BAA8B,CAC5C,MAAM,EACN,eAAe,EACf,MAAM,EACN,WAAW,EACX,OAAO,CAAC,UAAU,CACrB,CAAC;QAEF,IAAI,gBAAgB,GAAuB,GAAG,CAAC,MAAM,CAAC;QAEtD,4DAA4D;QAC5D,MAAM,YAAY,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC;QACtE,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1B,MAAM,cAAc,GAAG,MAAM,OAAO,CAAC,GAAG,CACpC,gBAAgB,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE;gBACvC,IAAI,IAAI,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;oBACxB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;oBACzB,wCAAwC;oBACxC,MAAM,UAAU,GAAG,KAAK,CAAC,OAAO,CAAC,6BAA6B,EAAE,EAAE,CAAC,CAAC;oBACpE,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;oBAEjD,oBAAoB;oBACpB,MAAM,EAAE,KAAK,EAAE,GAAG,IAAA,uBAAY,GAAE,CAAC,iBAAiB,CAAC;oBACnD,MAAM,EAAE,UAAU,EAAE,GAAG,IAAA,uBAAY,GAAE,CAAC;oBACtC,MAAM,QAAQ,GAAG,mBAAmB,KAAK,IAAI,UAAU,IAAI,KAAK,MAAM,CAAC;oBAEvE,2CAA2C;oBAC3C,MAAM,MAAM,GAAG,iBAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;oBAErC,MAAM,MAAM,GAAG,IAAI,uBAAgB,CAC/B,MAAM,EACN,QAAQ,EACR,WAAW,CACd,CAAC;oBAEF,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;oBACnD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAsB,CAAC;gBAC9D,CAAC;gBACD,OAAO,IAAI,CAAC;YAChB,CAAC,CAAC,CACL,CAAC;YACF,gBAAgB,GAAG,cAAc,CAAC;QACtC,CAAC;QAED,OAAO,IAAA,8BAAa,EAAC,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE;YACvC,KAAK,EAAE,GAAG,CAAC,EAAE;YACb,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,MAAM,EAAE,gBAAgB;SAC3B,CAAC,CAAC;IAEP,CAAC;IAAC,OAAO,KAAU,EAAE,CAAC;QAClB,cAAG,CAAC,KAAK,CAAC,iCAAiC,eAAe,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QACzE,IAAI,KAAK,CAAC,UAAU,KAAK,GAAG,IAAI,MAAM,CAAC,2BAA2B,EAAE,CAAC;YACjE,MAAM,IAAI,kCAAsB,CAAC,KAAK,CAAC,UAAU,EAAE,0CAA0C,CAAC,CAAC;QACnG,CAAC;aAAM,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,yCAAyC,CAAC,EAAE,CAAC;YAC3E,wCAAwC;YACxC,MAAM,IAAI,qCAAyB,CAAC,aAAa,EAAE,OAAO,CAAC,QAAQ,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;QACxF,CAAC;aAAM,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,qCAAqC,CAAC,EAAE,CAAC;YACvE,wCAAwC;YACxC,MAAM,IAAI,qCAAyB,CAAC,OAAO,EAAE,OAAO,CAAC,QAAQ,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;QAClF,CAAC;aAAM,CAAC;YACJ,MAAM,IAAI,KAAK,CAAC,gCAAgC,eAAe,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QACzF,CAAC;IACL,CAAC;AACL,CAAC;AAEM,KAAK,UAAU,8BAA8B,CAChD,MAAsB,EACtB,eAAuB,EACvB,MAAkC,EAClC,WAAgB,EAChB,KAAe;IAEf,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC;IAC7B,MAAM,IAAI,GAAG,IAAA,uBAAY,GAAE,CAAC;IAC5B,MAAM,KAAK,GAAG,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC;IAC3C,IAAI,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC;IACxB,IAAI,QAAQ,EAAE,CAAC;QACX,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IACjC,CAAC;IACD,MAAM,QAAQ,GAAyB;QACnC,MAAM,EAAE,IAAI,CAAC,iBAAiB,CAAC,KAAK;QACpC,WAAW,EAAE,IAAI,CAAC,iBAAiB,CAAC,UAAU;QAC9C,aAAa,EAAE,IAAI,CAAC,YAAY;KACnC,CAAC;IAEF,IAAI,0BAA0B,GAA6B,SAAS,CAAC;IACrE,IAAI,MAAM,CAAC,sBAAsB,EAAE,CAAC;QAChC,iCAAiC;QACjC,IAAI,IAAI,CAAC,OAAO,GAAG,CAAC,EAAE,CAAC;YACnB,cAAG,CAAC,IAAI,CAAC,sCAAsC,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;YACzE,MAAM,OAAO,GAAqB;gBAC9B,KAAK,EAAE,EAAE,gBAAgB,EAAE,CAAC,KAAK,CAAC,EAAE;gBACpC,KAAK,EAAE,CAAC;aACX,CAAC;YACF,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE;gBAC/D,cAAG,CAAC,IAAI,CAAC,gBAAgB,EAAE,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC;gBAC7C,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YACnD,CAAC,CAAC,CAAC;YAEH,IAAI,WAAW,EAAE,CAAC;gBACd,cAAG,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC;gBAChD,0BAA0B,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;YAC5E,CAAC;QACL,CAAC;IACL,CAAC;IACD,IAAI,KAAK,IAAI,0BAA0B,EAAE,KAAK,EAAE,CAAC;QAC7C,cAAG,CAAC,IAAI,CAAC,2BAA2B,EAAE,EAAE,KAAK,EAAE,0BAA0B,EAAE,KAAK,EAAE,CAAC,CAAC;IACxF,CAAC;IAED,MAAM,MAAM,GAAsC;QAC9C,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,aAAa,EAAE,MAAM,CAAC,aAAa;QACnC,WAAW,EAAE,MAAM,CAAC,eAAe;KACtC,CAAC;IACF,MAAM,IAAI,GAAG;QACT,GAAG,WAAW;QACd,cAAc,EAAE,0BAA0B,EAAE,KAAK;KACpD,CAAC;IAEF,MAAM,aAAa,GAAG,MAAM,CAAC,aAAa,CAAC;IAE3C,cAAG,CAAC,KAAK,CAAC,gCAAgC,eAAe,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;IAE9G,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,YAAY;SAChC,aAAa,CAAC,eAAe,EAAE;QAC5B,MAAM;QACN,IAAI;QACJ,aAAa;QACb,IAAI;QACJ,MAAM,EAAE,KAAK;QACb,QAAQ;KACX,CAAC;SACD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;QACX,cAAG,CAAC,KAAK,CAAC,+BAA+B,eAAe,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;QACrE,MAAM,GAAG,CAAC;IACd,CAAC,CAAC,CAAC;IAEP,IAAI,KAAK,EAAE,CAAC;QACR,cAAG,CAAC,IAAI,CAAC,wBAAwB,eAAe,EAAE,EAAE,GAAG,CAAC,CAAC;IAC7D,CAAC;IAED,IAAI,GAAG,CAAC,KAAK,IAAI,GAAG,CAAC,MAAM,KAAK,2BAAkB,CAAC,MAAM,EAAE,CAAC;QACxD,cAAG,CAAC,KAAK,CAAC,+BAA+B,eAAe,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;QAClF,MAAM,IAAI,KAAK,CAAC,gCAAgC,eAAe,KAAK,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;IACrF,CAAC;IAED,OAAO,GAAG,CAAC;AACf,CAAC"}
|
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.extractDocumentText = extractDocumentText;
|
|
4
|
+
const activity_1 = require("@temporalio/activity");
|
|
5
|
+
const markitdown_js_1 = require("../conversion/markitdown.js");
|
|
6
|
+
const mutool_js_1 = require("../conversion/mutool.js");
|
|
7
|
+
const pandoc_js_1 = require("../conversion/pandoc.js");
|
|
8
|
+
const ActivityContext_js_1 = require("../dsl/setup/ActivityContext.js");
|
|
9
|
+
const errors_js_1 = require("../errors.js");
|
|
10
|
+
const result_types_js_1 = require("../result-types.js");
|
|
11
|
+
const blobs_js_1 = require("../utils/blobs.js");
|
|
12
|
+
const text_preview_utils_js_1 = require("../utils/text-preview-utils.js");
|
|
13
|
+
const tokens_js_1 = require("../utils/tokens.js");
|
|
14
|
+
//@ts-ignore
|
|
15
|
+
const JSON = {
|
|
16
|
+
name: "extractDocumentText",
|
|
17
|
+
};
|
|
18
|
+
async function extractDocumentText(payload) {
|
|
19
|
+
const context = await (0, ActivityContext_js_1.setupActivity)(payload);
|
|
20
|
+
const { client, inputType, params } = context;
|
|
21
|
+
const { output_storage_path } = params;
|
|
22
|
+
if (inputType === 'files') {
|
|
23
|
+
// File mode: extract from file source
|
|
24
|
+
if (!output_storage_path) {
|
|
25
|
+
throw new Error('output_storage_path is required when extracting text from file sources');
|
|
26
|
+
}
|
|
27
|
+
return extractFromFileSource(client, context.file, output_storage_path);
|
|
28
|
+
}
|
|
29
|
+
else {
|
|
30
|
+
// Object mode: fetch from object store
|
|
31
|
+
return extractFromObject(client, context.objectId, context.objectIds || []);
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
async function extractFromObject(client, objectId, objectIds) {
|
|
35
|
+
const r = await client.objects.find({
|
|
36
|
+
query: { _id: objectId },
|
|
37
|
+
limit: 1,
|
|
38
|
+
select: "+text",
|
|
39
|
+
});
|
|
40
|
+
const doc = r[0];
|
|
41
|
+
if (!doc) {
|
|
42
|
+
activity_1.log.error(`Document ${objectId} not found`);
|
|
43
|
+
throw new errors_js_1.DocumentNotFoundError(`Document ${objectId} not found`, objectIds);
|
|
44
|
+
}
|
|
45
|
+
activity_1.log.info(`Extracting text for object ${doc.id}`);
|
|
46
|
+
if (!doc.content?.type || !doc.content?.source) {
|
|
47
|
+
if (doc.text) {
|
|
48
|
+
return createResponse(doc, doc.text, result_types_js_1.TextExtractionStatus.skipped, "Text present and no source or type");
|
|
49
|
+
}
|
|
50
|
+
else {
|
|
51
|
+
return createResponse(doc, "", result_types_js_1.TextExtractionStatus.error, "No source or type found");
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
//skip if text already extracted and proper etag
|
|
55
|
+
if (doc.text && doc.text.length > 0 && doc.text_etag === doc.content.etag) {
|
|
56
|
+
return createResponse(doc, doc.text, result_types_js_1.TextExtractionStatus.skipped, "Text already extracted");
|
|
57
|
+
}
|
|
58
|
+
let fileBuffer;
|
|
59
|
+
try {
|
|
60
|
+
fileBuffer = await (0, blobs_js_1.fetchBlobAsBuffer)(client, doc.content.source);
|
|
61
|
+
}
|
|
62
|
+
catch (e) {
|
|
63
|
+
activity_1.log.error(`Error reading file: ${e}`);
|
|
64
|
+
return createResponse(doc, "", result_types_js_1.TextExtractionStatus.error, e.message);
|
|
65
|
+
}
|
|
66
|
+
const txt = await extractTextFromBuffer(fileBuffer, doc.content.type);
|
|
67
|
+
if (!txt) {
|
|
68
|
+
return createResponse(doc, doc.text ?? "", result_types_js_1.TextExtractionStatus.skipped, `Unsupported mime type: ${doc.content.type}`);
|
|
69
|
+
}
|
|
70
|
+
const tokensData = (0, tokens_js_1.countTokens)(txt);
|
|
71
|
+
const etag = doc.content.etag ?? (0, blobs_js_1.md5)(txt);
|
|
72
|
+
const updateData = {
|
|
73
|
+
text: txt,
|
|
74
|
+
text_etag: etag,
|
|
75
|
+
tokens: {
|
|
76
|
+
...tokensData,
|
|
77
|
+
etag: etag,
|
|
78
|
+
},
|
|
79
|
+
};
|
|
80
|
+
await client.objects.update(doc.id, updateData);
|
|
81
|
+
return createResponse(doc, txt, result_types_js_1.TextExtractionStatus.success);
|
|
82
|
+
}
|
|
83
|
+
async function extractFromFileSource(client, input_file, output_storage_path) {
|
|
84
|
+
activity_1.log.info(`Extracting text from ${input_file}`);
|
|
85
|
+
let fileBuffer;
|
|
86
|
+
try {
|
|
87
|
+
fileBuffer = await (0, blobs_js_1.fetchBlobAsBuffer)(client, input_file.url);
|
|
88
|
+
}
|
|
89
|
+
catch (e) {
|
|
90
|
+
activity_1.log.error(`Error reading file: ${e}`);
|
|
91
|
+
return (0, text_preview_utils_js_1.createFileSourceResult)(input_file.url, output_storage_path, null);
|
|
92
|
+
}
|
|
93
|
+
const txt = await extractTextFromBuffer(fileBuffer, input_file.mimetype);
|
|
94
|
+
// Upload extracted text to storage
|
|
95
|
+
if (txt && output_storage_path) {
|
|
96
|
+
await (0, text_preview_utils_js_1.uploadTextPreviewToStorage)(client, txt, output_storage_path, "Document");
|
|
97
|
+
}
|
|
98
|
+
return (0, text_preview_utils_js_1.createFileSourceResult)(input_file.url, output_storage_path, txt);
|
|
99
|
+
}
|
|
100
|
+
async function extractTextFromBuffer(fileBuffer, mimeType) {
|
|
101
|
+
let txt;
|
|
102
|
+
switch (mimeType) {
|
|
103
|
+
case "application/pdf":
|
|
104
|
+
txt = await (0, mutool_js_1.mutoolPdfToText)(fileBuffer);
|
|
105
|
+
break;
|
|
106
|
+
case "text/plain":
|
|
107
|
+
txt = fileBuffer.toString("utf8");
|
|
108
|
+
break;
|
|
109
|
+
//docx
|
|
110
|
+
case "application/vnd.openxmlformats-officedocument.wordprocessingml.document":
|
|
111
|
+
txt = await (0, markitdown_js_1.markdownWithMarkitdown)(fileBuffer, "docx");
|
|
112
|
+
break;
|
|
113
|
+
//pptx
|
|
114
|
+
case "application/vnd.openxmlformats-officedocument.presentationml.presentation":
|
|
115
|
+
txt = await (0, markitdown_js_1.markdownWithMarkitdown)(fileBuffer, "pptx");
|
|
116
|
+
break;
|
|
117
|
+
//html
|
|
118
|
+
case "text/html":
|
|
119
|
+
txt = await (0, pandoc_js_1.markdownWithPandoc)(fileBuffer, "html");
|
|
120
|
+
break;
|
|
121
|
+
//opendocument
|
|
122
|
+
case "application/vnd.oasis.opendocument.text":
|
|
123
|
+
txt = await (0, pandoc_js_1.markdownWithPandoc)(fileBuffer, "odt");
|
|
124
|
+
break;
|
|
125
|
+
//rtf
|
|
126
|
+
case "application/rtf":
|
|
127
|
+
txt = await (0, pandoc_js_1.markdownWithPandoc)(fileBuffer, "rtf");
|
|
128
|
+
break;
|
|
129
|
+
//markdown
|
|
130
|
+
case "text/markdown":
|
|
131
|
+
txt = fileBuffer.toString("utf8");
|
|
132
|
+
break;
|
|
133
|
+
//csv
|
|
134
|
+
case "text/csv":
|
|
135
|
+
txt = fileBuffer.toString("utf8");
|
|
136
|
+
break;
|
|
137
|
+
//typescript
|
|
138
|
+
case "application/typescript":
|
|
139
|
+
txt = fileBuffer.toString("utf8");
|
|
140
|
+
break;
|
|
141
|
+
//javascript
|
|
142
|
+
case "application/javascript":
|
|
143
|
+
txt = fileBuffer.toString("utf8");
|
|
144
|
+
break;
|
|
145
|
+
//json
|
|
146
|
+
case "application/json":
|
|
147
|
+
txt = fileBuffer.toString("utf8");
|
|
148
|
+
break;
|
|
149
|
+
default:
|
|
150
|
+
if (sniffIfText(fileBuffer)) {
|
|
151
|
+
txt = fileBuffer.toString("utf8"); //TODO: add charset detection
|
|
152
|
+
break;
|
|
153
|
+
}
|
|
154
|
+
return null;
|
|
155
|
+
}
|
|
156
|
+
return txt;
|
|
157
|
+
}
|
|
158
|
+
function createResponse(doc, text, status, message) {
|
|
159
|
+
return {
|
|
160
|
+
status,
|
|
161
|
+
message,
|
|
162
|
+
tokens: doc.tokens,
|
|
163
|
+
len: text.length,
|
|
164
|
+
objectId: doc.id,
|
|
165
|
+
hasText: !!text,
|
|
166
|
+
};
|
|
167
|
+
}
|
|
168
|
+
function sniffIfText(buf) {
|
|
169
|
+
// If file is too large, don't even try
|
|
170
|
+
if (buf.length > 500 * 1024) {
|
|
171
|
+
return false;
|
|
172
|
+
}
|
|
173
|
+
// Count binary/control characters
|
|
174
|
+
let binaryCount = 0;
|
|
175
|
+
const sampleSize = Math.min(buf.length, 1000); // Check first 1000 bytes
|
|
176
|
+
for (let i = 0; i < sampleSize; i++) {
|
|
177
|
+
// Count control characters (except common whitespace)
|
|
178
|
+
const byte = buf[i];
|
|
179
|
+
if ((byte < 32 && ![9, 10, 13].includes(byte)) || byte === 0) {
|
|
180
|
+
binaryCount++;
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
// If more than 10% binary/control chars, probably not text
|
|
184
|
+
if (binaryCount / sampleSize > 0.1) {
|
|
185
|
+
return false;
|
|
186
|
+
}
|
|
187
|
+
// Additional check for valid UTF-8 encoding
|
|
188
|
+
try {
|
|
189
|
+
const s = buf.toString("utf8");
|
|
190
|
+
return s.length > 0 && !s.includes("\uFFFD"); // Replacement character
|
|
191
|
+
}
|
|
192
|
+
catch (e) {
|
|
193
|
+
return false;
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
//# sourceMappingURL=extractDocumentText.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"extractDocumentText.js","sourceRoot":"","sources":["../../../src/activities/extractDocumentText.ts"],"names":[],"mappings":";;AAkCA,kDAiBC;AAnDD,mDAA2C;AAQ3C,+DAAqE;AACrE,uDAA0D;AAC1D,uDAA6D;AAC7D,wEAAgE;AAChE,4CAAqD;AACrD,wDAAgF;AAChF,gDAA2D;AAC3D,0EAGwC;AACxC,kDAAiD;AAEjD,YAAY;AACZ,MAAM,IAAI,GAAoB;IAC1B,IAAI,EAAE,qBAAqB;CAC9B,CAAC;AAUK,KAAK,UAAU,mBAAmB,CACrC,OAA+D;IAE/D,MAAM,OAAO,GAAG,MAAM,IAAA,kCAAa,EAAC,OAAO,CAAC,CAAC;IAC7C,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAC9C,MAAM,EAAE,mBAAmB,EAAE,GAAG,MAAM,CAAC;IAEvC,IAAI,SAAS,KAAK,OAAO,EAAE,CAAC;QACxB,sCAAsC;QACtC,IAAI,CAAC,mBAAmB,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,wEAAwE,CAAC,CAAC;QAC9F,CAAC;QACD,OAAO,qBAAqB,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,EAAE,mBAAmB,CAAC,CAAC;IAC5E,CAAC;SAAM,CAAC;QACJ,uCAAuC;QACvC,OAAO,iBAAiB,CAAC,MAAM,EAAE,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC;IAChF,CAAC;AACL,CAAC;AAED,KAAK,UAAU,iBAAiB,CAC5B,MAAW,EACX,QAAgB,EAChB,SAAmB;IAEnB,MAAM,CAAC,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC;QAChC,KAAK,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE;QACxB,KAAK,EAAE,CAAC;QACR,MAAM,EAAE,OAAO;KAClB,CAAC,CAAC;IACH,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,CAAkB,CAAC;IAClC,IAAI,CAAC,GAAG,EAAE,CAAC;QACP,cAAG,CAAC,KAAK,CAAC,YAAY,QAAQ,YAAY,CAAC,CAAC;QAC5C,MAAM,IAAI,iCAAqB,CAAC,YAAY,QAAQ,YAAY,EAAE,SAAS,CAAC,CAAC;IACjF,CAAC;IAED,cAAG,CAAC,IAAI,CAAC,8BAA8B,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC;IAEjD,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;QAC7C,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;YACX,OAAO,cAAc,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE,sCAAoB,CAAC,OAAO,EAAE,oCAAoC,CAAC,CAAC;QAC7G,CAAC;aAAM,CAAC;YACJ,OAAO,cAAc,CAAC,GAAG,EAAE,EAAE,EAAE,sCAAoB,CAAC,KAAK,EAAE,yBAAyB,CAAC,CAAC;QAC1F,CAAC;IACL,CAAC;IAED,gDAAgD;IAChD,IAAI,GAAG,CAAC,IAAI,IAAI,GAAG,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,GAAG,CAAC,SAAS,KAAK,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QACxE,OAAO,cAAc,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE,sCAAoB,CAAC,OAAO,EAAE,wBAAwB,CAAC,CAAC;IACjG,CAAC;IAED,IAAI,UAAkB,CAAC;IACvB,IAAI,CAAC;QACD,UAAU,GAAG,MAAM,IAAA,4BAAiB,EAAC,MAAM,EAAE,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACrE,CAAC;IAAC,OAAO,CAAM,EAAE,CAAC;QACd,cAAG,CAAC,KAAK,CAAC,uBAAuB,CAAC,EAAE,CAAC,CAAC;QACtC,OAAO,cAAc,CAAC,GAAG,EAAE,EAAE,EAAE,sCAAoB,CAAC,KAAK,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC;IAC1E,CAAC;IAED,MAAM,GAAG,GAAG,MAAM,qBAAqB,CAAC,UAAU,EAAE,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACtE,IAAI,CAAC,GAAG,EAAE,CAAC;QACP,OAAO,cAAc,CACjB,GAAG,EACH,GAAG,CAAC,IAAI,IAAI,EAAE,EACd,sCAAoB,CAAC,OAAO,EAC5B,0BAA0B,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAC/C,CAAC;IACN,CAAC;IAED,MAAM,UAAU,GAAG,IAAA,uBAAW,EAAC,GAAG,CAAC,CAAC;IACpC,MAAM,IAAI,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,IAAI,IAAA,cAAG,EAAC,GAAG,CAAC,CAAC;IAE1C,MAAM,UAAU,GAA+B;QAC3C,IAAI,EAAE,GAAG;QACT,SAAS,EAAE,IAAI;QACf,MAAM,EAAE;YACJ,GAAG,UAAU;YACb,IAAI,EAAE,IAAI;SACb;KACJ,CAAC;IAEF,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;IAEhD,OAAO,cAAc,CAAC,GAAG,EAAE,GAAG,EAAE,sCAAoB,CAAC,OAAO,CAAC,CAAC;AAClE,CAAC;AAED,KAAK,UAAU,qBAAqB,CAChC,MAAW,EACX,UAA6B,EAC7B,mBAA2B;IAE3B,cAAG,CAAC,IAAI,CAAC,wBAAwB,UAAU,EAAE,CAAC,CAAC;IAE/C,IAAI,UAAkB,CAAC;IACvB,IAAI,CAAC;QACD,UAAU,GAAG,MAAM,IAAA,4BAAiB,EAAC,MAAM,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC;IACjE,CAAC;IAAC,OAAO,CAAM,EAAE,CAAC;QACd,cAAG,CAAC,KAAK,CAAC,uBAAuB,CAAC,EAAE,CAAC,CAAC;QACtC,OAAO,IAAA,8CAAsB,EAAC,UAAU,CAAC,GAAG,EAAE,mBAAmB,EAAE,IAAI,CAAC,CAAC;IAC7E,CAAC;IAED,MAAM,GAAG,GAAG,MAAM,qBAAqB,CAAC,UAAU,EAAE,UAAU,CAAC,QAAQ,CAAC,CAAC;IAEzE,mCAAmC;IACnC,IAAI,GAAG,IAAI,mBAAmB,EAAE,CAAC;QAC7B,MAAM,IAAA,kDAA0B,EAAC,MAAM,EAAE,GAAG,EAAE,mBAAmB,EAAE,UAAU,CAAC,CAAC;IACnF,CAAC;IAED,OAAO,IAAA,8CAAsB,EAAC,UAAU,CAAC,GAAG,EAAE,mBAAmB,EAAE,GAAG,CAAC,CAAC;AAC5E,CAAC;AAED,KAAK,UAAU,qBAAqB,CAAC,UAAkB,EAAE,QAAgB;IACrE,IAAI,GAAW,CAAC;IAEhB,QAAQ,QAAQ,EAAE,CAAC;QACf,KAAK,iBAAiB;YAClB,GAAG,GAAG,MAAM,IAAA,2BAAe,EAAC,UAAU,CAAC,CAAC;YACxC,MAAM;QAEV,KAAK,YAAY;YACb,GAAG,GAAG,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAClC,MAAM;QAEV,MAAM;QACN,KAAK,yEAAyE;YAC1E,GAAG,GAAG,MAAM,IAAA,sCAAsB,EAAC,UAAU,EAAE,MAAM,CAAC,CAAC;YACvD,MAAM;QAEV,MAAM;QACN,KAAK,2EAA2E;YAC5E,GAAG,GAAG,MAAM,IAAA,sCAAsB,EAAC,UAAU,EAAE,MAAM,CAAC,CAAC;YACvD,MAAM;QAEV,MAAM;QACN,KAAK,WAAW;YACZ,GAAG,GAAG,MAAM,IAAA,8BAAkB,EAAC,UAAU,EAAE,MAAM,CAAC,CAAC;YACnD,MAAM;QAEV,cAAc;QACd,KAAK,yCAAyC;YAC1C,GAAG,GAAG,MAAM,IAAA,8BAAkB,EAAC,UAAU,EAAE,KAAK,CAAC,CAAC;YAClD,MAAM;QAEV,KAAK;QACL,KAAK,iBAAiB;YAClB,GAAG,GAAG,MAAM,IAAA,8BAAkB,EAAC,UAAU,EAAE,KAAK,CAAC,CAAC;YAClD,MAAM;QAEV,UAAU;QACV,KAAK,eAAe;YAChB,GAAG,GAAG,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAClC,MAAM;QAEV,KAAK;QACL,KAAK,UAAU;YACX,GAAG,GAAG,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAClC,MAAM;QAEV,YAAY;QACZ,KAAK,wBAAwB;YACzB,GAAG,GAAG,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAClC,MAAM;QAEV,YAAY;QACZ,KAAK,wBAAwB;YACzB,GAAG,GAAG,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAClC,MAAM;QAEV,MAAM;QACN,KAAK,kBAAkB;YACnB,GAAG,GAAG,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAClC,MAAM;QAEV;YACI,IAAI,WAAW,CAAC,UAAU,CAAC,EAAE,CAAC;gBAC1B,GAAG,GAAG,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,6BAA6B;gBAChE,MAAM;YACV,CAAC;YACD,OAAO,IAAI,CAAC;IACpB,CAAC;IAED,OAAO,GAAG,CAAC;AACf,CAAC;AAED,SAAS,cAAc,CACnB,GAAkB,EAClB,IAAY,EACZ,MAA4B,EAC5B,OAAgB;IAEhB,OAAO;QACH,MAAM;QACN,OAAO;QACP,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,GAAG,EAAE,IAAI,CAAC,MAAM;QAChB,QAAQ,EAAE,GAAG,CAAC,EAAE;QAChB,OAAO,EAAE,CAAC,CAAC,IAAI;KAClB,CAAC;AACN,CAAC;AAED,SAAS,WAAW,CAAC,GAAW;IAC5B,uCAAuC;IACvC,IAAI,GAAG,CAAC,MAAM,GAAG,GAAG,GAAG,IAAI,EAAE,CAAC;QAC1B,OAAO,KAAK,CAAC;IACjB,CAAC;IAED,kCAAkC;IAClC,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,yBAAyB;IAExE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,sDAAsD;QACtD,MAAM,IAAI,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;QACpB,IAAI,CAAC,IAAI,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;YAC3D,WAAW,EAAE,CAAC;QAClB,CAAC;IACL,CAAC;IAED,2DAA2D;IAC3D,IAAI,WAAW,GAAG,UAAU,GAAG,GAAG,EAAE,CAAC;QACjC,OAAO,KAAK,CAAC;IACjB,CAAC;IAED,4CAA4C;IAC5C,IAAI,CAAC;QACD,MAAM,CAAC,GAAG,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAC/B,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,wBAAwB;IAC1E,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,OAAO,KAAK,CAAC;IACjB,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.generateDocumentProperties = generateDocumentProperties;
|
|
4
|
+
const activity_1 = require("@temporalio/activity");
|
|
5
|
+
const ActivityContext_js_1 = require("../dsl/setup/ActivityContext.js");
|
|
6
|
+
const tokens_js_1 = require("../utils/tokens.js");
|
|
7
|
+
const executeInteraction_js_1 = require("./executeInteraction.js");
|
|
8
|
+
const INT_EXTRACT_INFORMATION = "sys:ExtractInformation";
|
|
9
|
+
async function generateDocumentProperties(payload) {
|
|
10
|
+
const context = await (0, ActivityContext_js_1.setupActivity)(payload);
|
|
11
|
+
const { params, client, objectId } = context;
|
|
12
|
+
const interactionName = params.interactionName ?? INT_EXTRACT_INFORMATION;
|
|
13
|
+
const project = await context.fetchProject();
|
|
14
|
+
const doc = await client.objects.retrieve(objectId, "+text");
|
|
15
|
+
const type = doc.type
|
|
16
|
+
? await client.types.catalog.resolve(doc.type)
|
|
17
|
+
: undefined;
|
|
18
|
+
if (!doc?.text && !params.use_vision && !doc?.content?.type?.startsWith("image/")) {
|
|
19
|
+
activity_1.log.warn(`Object ${objectId} not found or text is empty`);
|
|
20
|
+
return { status: "failed", error: "no-text" };
|
|
21
|
+
}
|
|
22
|
+
if (!type || !type.object_schema) {
|
|
23
|
+
activity_1.log.info(`Object ${objectId} has no schema`);
|
|
24
|
+
return { document: objectId, status: "skipped", message: "no schema defined on type" };
|
|
25
|
+
}
|
|
26
|
+
const getImageRef = () => {
|
|
27
|
+
if (doc.content?.type?.startsWith("image/")) {
|
|
28
|
+
return "store:" + doc.id;
|
|
29
|
+
}
|
|
30
|
+
if (params.use_vision && doc.content?.type?.startsWith("application/pdf")) {
|
|
31
|
+
return "store:" + doc.id;
|
|
32
|
+
}
|
|
33
|
+
activity_1.log.info(`Object ${objectId} is not an image or pdf`);
|
|
34
|
+
return undefined;
|
|
35
|
+
};
|
|
36
|
+
const content = doc.text
|
|
37
|
+
? (0, tokens_js_1.truncByMaxTokens)(doc.text, params.truncate || 30000)
|
|
38
|
+
: undefined;
|
|
39
|
+
const promptData = {
|
|
40
|
+
content: content,
|
|
41
|
+
image: getImageRef() ?? undefined,
|
|
42
|
+
human_context: project?.configuration?.human_context ?? undefined,
|
|
43
|
+
};
|
|
44
|
+
activity_1.log.info(` Extracting information from object ${objectId} with type ${type.name}`, payload.debug_mode ? { params } : undefined);
|
|
45
|
+
const infoRes = await (0, executeInteraction_js_1.executeInteractionFromActivity)(client, interactionName, {
|
|
46
|
+
...params,
|
|
47
|
+
include_previous_error: true,
|
|
48
|
+
result_schema: type.object_schema,
|
|
49
|
+
validate_result: type.strict_mode,
|
|
50
|
+
}, promptData, payload.debug_mode ?? false);
|
|
51
|
+
const getText = () => {
|
|
52
|
+
if (doc.text) {
|
|
53
|
+
return undefined;
|
|
54
|
+
}
|
|
55
|
+
let text = "";
|
|
56
|
+
const jsonResult = infoRes.result.object();
|
|
57
|
+
if (jsonResult.title) {
|
|
58
|
+
text += jsonResult.title + "\n";
|
|
59
|
+
}
|
|
60
|
+
if (jsonResult.description) {
|
|
61
|
+
text += jsonResult.description;
|
|
62
|
+
}
|
|
63
|
+
if (text) {
|
|
64
|
+
return text;
|
|
65
|
+
}
|
|
66
|
+
else {
|
|
67
|
+
return undefined;
|
|
68
|
+
}
|
|
69
|
+
};
|
|
70
|
+
activity_1.log.info(`Extracted information from object ${objectId} with type ${type.name}`, { runId: infoRes.id });
|
|
71
|
+
await client.objects.update(doc.id, {
|
|
72
|
+
properties: {
|
|
73
|
+
...infoRes.result.object(),
|
|
74
|
+
etag: doc.text_etag,
|
|
75
|
+
},
|
|
76
|
+
text: getText(),
|
|
77
|
+
generation_run_info: {
|
|
78
|
+
id: infoRes.id,
|
|
79
|
+
date: new Date().toISOString(),
|
|
80
|
+
model: infoRes.modelId ?? "",
|
|
81
|
+
},
|
|
82
|
+
}, { suppressWorkflows: true });
|
|
83
|
+
return { status: "completed" };
|
|
84
|
+
}
|
|
85
|
+
//# sourceMappingURL=generateDocumentProperties.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"generateDocumentProperties.js","sourceRoot":"","sources":["../../../src/activities/generateDocumentProperties.ts"],"names":[],"mappings":";;AAsBA,gEAmGC;AAzHD,mDAA2C;AAE3C,wEAAgE;AAChE,kDAAoE;AACpE,mEAAqG;AAErG,MAAM,uBAAuB,GAAG,wBAAwB,CAAC;AAgBlD,KAAK,UAAU,0BAA0B,CAC5C,OAAsE;IAEtE,MAAM,OAAO,GAAG,MAAM,IAAA,kCAAa,EAAmC,OAAO,CAAC,CAAC;IAC/E,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;IAC7C,MAAM,eAAe,GAAG,MAAM,CAAC,eAAe,IAAI,uBAAuB,CAAC;IAE1E,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,YAAY,EAAE,CAAC;IAE7C,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC7D,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI;QACjB,CAAC,CAAC,MAAM,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC;QAC9C,CAAC,CAAC,SAAS,CAAC;IAEhB,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,CAAC,UAAU,IAAI,CAAC,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QAChF,cAAG,CAAC,IAAI,CAAC,UAAU,QAAQ,6BAA6B,CAAC,CAAC;QAC1D,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;IAClD,CAAC;IAED,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;QAC/B,cAAG,CAAC,IAAI,CAAC,UAAU,QAAQ,gBAAgB,CAAC,CAAC;QAC7C,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,2BAA2B,EAAE,CAAC;IAC3F,CAAC;IAED,MAAM,WAAW,GAAG,GAAG,EAAE;QACrB,IAAI,GAAG,CAAC,OAAO,EAAE,IAAI,EAAE,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC1C,OAAO,QAAQ,GAAG,GAAG,CAAC,EAAE,CAAC;QAC7B,CAAC;QAED,IAAI,MAAM,CAAC,UAAU,IAAI,GAAG,CAAC,OAAO,EAAE,IAAI,EAAE,UAAU,CAAC,iBAAiB,CAAC,EAAE,CAAC;YACxE,OAAO,QAAQ,GAAG,GAAG,CAAC,EAAE,CAAC;QAC7B,CAAC;QAED,cAAG,CAAC,IAAI,CAAC,UAAU,QAAQ,yBAAyB,CAAC,CAAC;QACtD,OAAO,SAAS,CAAC;IACrB,CAAC,CAAC;IAEF,MAAM,OAAO,GAAG,GAAG,CAAC,IAAI;QACpB,CAAC,CAAC,IAAA,4BAAgB,EAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,QAAQ,IAAI,KAAK,CAAC;QACtD,CAAC,CAAC,SAAS,CAAC;IAEhB,MAAM,UAAU,GAAG;QACf,OAAO,EAAE,OAAO;QAChB,KAAK,EAAE,WAAW,EAAE,IAAI,SAAS;QACjC,aAAa,EAAE,OAAO,EAAE,aAAa,EAAE,aAAa,IAAI,SAAS;KACpE,CAAC;IAEF,cAAG,CAAC,IAAI,CACJ,uCAAuC,QAAQ,cAAc,IAAI,CAAC,IAAI,EAAE,EACxE,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,SAAS,CAC9C,CAAC;IAEF,MAAM,OAAO,GAAG,MAAM,IAAA,sDAA8B,EAChD,MAAM,EACN,eAAe,EACf;QACI,GAAG,MAAM;QACT,sBAAsB,EAAE,IAAI;QAC5B,aAAa,EAAE,IAAI,CAAC,aAAa;QACjC,eAAe,EAAE,IAAI,CAAC,WAAW;KACpC,EACD,UAAU,EACV,OAAO,CAAC,UAAU,IAAI,KAAK,CAC9B,CAAC;IAEF,MAAM,OAAO,GAAG,GAAG,EAAE;QACjB,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;YACX,OAAO,SAAS,CAAC;QACrB,CAAC;QACD,IAAI,IAAI,GAAG,EAAE,CAAC;QACd,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QAC3C,IAAI,UAAU,CAAC,KAAK,EAAE,CAAC;YACnB,IAAI,IAAI,UAAU,CAAC,KAAK,GAAG,IAAI,CAAC;QACpC,CAAC;QACD,IAAI,UAAU,CAAC,WAAW,EAAE,CAAC;YACzB,IAAI,IAAI,UAAU,CAAC,WAAW,CAAC;QACnC,CAAC;QACD,IAAI,IAAI,EAAE,CAAC;YACP,OAAO,IAAI,CAAC;QAChB,CAAC;aAAM,CAAC;YACJ,OAAO,SAAS,CAAC;QACrB,CAAC;IACL,CAAC,CAAC;IAEF,cAAG,CAAC,IAAI,CAAC,qCAAqC,QAAQ,cAAc,IAAI,CAAC,IAAI,EAAE,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC;IACxG,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE;QAChC,UAAU,EAAE;YACR,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE;YAC1B,IAAI,EAAE,GAAG,CAAC,SAAS;SACtB;QACD,IAAI,EAAE,OAAO,EAAE;QACf,mBAAmB,EAAE;YACjB,EAAE,EAAE,OAAO,CAAC,EAAE;YACd,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YAC9B,KAAK,EAAE,OAAO,CAAC,OAAO,IAAI,EAAE;SAC/B;KACJ,EAAE,EAAE,iBAAiB,EAAE,IAAI,EAAE,CAAC,CAAC;IAEhC,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC;AACnC,CAAC"}
|
|
@@ -0,0 +1,267 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.generateEmbeddings = generateEmbeddings;
|
|
4
|
+
const activity_1 = require("@temporalio/activity");
|
|
5
|
+
const client_1 = require("@vertesia/client");
|
|
6
|
+
const common_1 = require("@vertesia/common");
|
|
7
|
+
const ActivityContext_js_1 = require("../dsl/setup/ActivityContext.js");
|
|
8
|
+
const errors_js_1 = require("../errors.js");
|
|
9
|
+
const blobs_js_1 = require("../utils/blobs.js");
|
|
10
|
+
const tokens_js_1 = require("../utils/tokens.js");
|
|
11
|
+
async function generateEmbeddings(payload) {
|
|
12
|
+
const { params, client, objectId, fetchProject } = await (0, ActivityContext_js_1.setupActivity)(payload);
|
|
13
|
+
const { force, type } = params;
|
|
14
|
+
const projectData = await fetchProject();
|
|
15
|
+
const config = projectData?.configuration.embeddings[type];
|
|
16
|
+
if (!projectData) {
|
|
17
|
+
throw new errors_js_1.DocumentNotFoundError("Project not found", [payload.project_id]);
|
|
18
|
+
}
|
|
19
|
+
if (!config) {
|
|
20
|
+
throw new errors_js_1.DocumentNotFoundError("Embeddings configuration not found", [
|
|
21
|
+
objectId,
|
|
22
|
+
]);
|
|
23
|
+
}
|
|
24
|
+
if (!projectData) {
|
|
25
|
+
throw new errors_js_1.DocumentNotFoundError("Project not found", [payload.project_id]);
|
|
26
|
+
}
|
|
27
|
+
if (!projectData?.configuration.embeddings[type]?.enabled) {
|
|
28
|
+
activity_1.log.debug(`Embeddings generation disabled for type ${type} on project: ${projectData.name} (${projectData.namespace})`, { config });
|
|
29
|
+
return {
|
|
30
|
+
id: objectId,
|
|
31
|
+
status: "skipped",
|
|
32
|
+
message: `Embeddings generation disabled for type ${type}`,
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
activity_1.log.debug(`${type} embedding generation starting for object ${objectId}`, {
|
|
36
|
+
force,
|
|
37
|
+
config,
|
|
38
|
+
});
|
|
39
|
+
if (!config.environment) {
|
|
40
|
+
throw new Error("No environment found in project configuration. Set environment in project configuration to generate embeddings.");
|
|
41
|
+
}
|
|
42
|
+
let document;
|
|
43
|
+
try {
|
|
44
|
+
document = await client.objects.retrieve(objectId, "+text +parts +embeddings +tokens +properties");
|
|
45
|
+
}
|
|
46
|
+
catch (error) {
|
|
47
|
+
if (error instanceof client_1.ZenoClientNotFoundError) {
|
|
48
|
+
throw new errors_js_1.DocumentNotFoundError(`Document not found: ${objectId}`, [objectId]);
|
|
49
|
+
}
|
|
50
|
+
throw error;
|
|
51
|
+
}
|
|
52
|
+
if (!document) {
|
|
53
|
+
throw new errors_js_1.DocumentNotFoundError("Document not found", [objectId]);
|
|
54
|
+
}
|
|
55
|
+
if (!document.content) {
|
|
56
|
+
throw new errors_js_1.DocumentNotFoundError("Document content not found", [objectId]);
|
|
57
|
+
}
|
|
58
|
+
let res;
|
|
59
|
+
switch (type) {
|
|
60
|
+
case common_1.SupportedEmbeddingTypes.text:
|
|
61
|
+
res = await generateTextEmbeddings({
|
|
62
|
+
client,
|
|
63
|
+
config,
|
|
64
|
+
document,
|
|
65
|
+
type,
|
|
66
|
+
force,
|
|
67
|
+
});
|
|
68
|
+
break;
|
|
69
|
+
case common_1.SupportedEmbeddingTypes.properties:
|
|
70
|
+
res = await generateTextEmbeddings({
|
|
71
|
+
client,
|
|
72
|
+
config,
|
|
73
|
+
document,
|
|
74
|
+
type,
|
|
75
|
+
force,
|
|
76
|
+
});
|
|
77
|
+
break;
|
|
78
|
+
case common_1.SupportedEmbeddingTypes.image:
|
|
79
|
+
res = await generateImageEmbeddings({
|
|
80
|
+
client,
|
|
81
|
+
config,
|
|
82
|
+
document,
|
|
83
|
+
type,
|
|
84
|
+
force,
|
|
85
|
+
});
|
|
86
|
+
break;
|
|
87
|
+
default:
|
|
88
|
+
res = {
|
|
89
|
+
id: objectId,
|
|
90
|
+
status: "failed",
|
|
91
|
+
message: `unsupported embedding type: ${type}`,
|
|
92
|
+
};
|
|
93
|
+
}
|
|
94
|
+
return res;
|
|
95
|
+
}
|
|
96
|
+
async function generateTextEmbeddings({ document, client, type, config, force }) {
|
|
97
|
+
if (!document) {
|
|
98
|
+
return { status: "error", message: "document is null or undefined" };
|
|
99
|
+
}
|
|
100
|
+
if (type !== common_1.SupportedEmbeddingTypes.text &&
|
|
101
|
+
type !== common_1.SupportedEmbeddingTypes.properties) {
|
|
102
|
+
return {
|
|
103
|
+
id: document.id,
|
|
104
|
+
status: "failed",
|
|
105
|
+
message: `unsupported embedding type: ${type}`,
|
|
106
|
+
};
|
|
107
|
+
}
|
|
108
|
+
if (type === common_1.SupportedEmbeddingTypes.text && !document.text) {
|
|
109
|
+
return { id: document.id, status: "failed", message: "no text found" };
|
|
110
|
+
}
|
|
111
|
+
if (type === common_1.SupportedEmbeddingTypes.properties && !document?.properties) {
|
|
112
|
+
return {
|
|
113
|
+
id: document.id,
|
|
114
|
+
status: "failed",
|
|
115
|
+
message: "no properties found",
|
|
116
|
+
};
|
|
117
|
+
}
|
|
118
|
+
const { environment } = config;
|
|
119
|
+
// Compute text etag for comparison
|
|
120
|
+
const textEtag = document.text_etag ?? (document.text ? (0, blobs_js_1.md5)(document.text) : undefined);
|
|
121
|
+
// Skip if embeddings already exist with matching etag (unless force=true)
|
|
122
|
+
const existingEmbedding = document.embeddings?.[type];
|
|
123
|
+
if (!force && existingEmbedding?.etag && textEtag && existingEmbedding.etag === textEtag) {
|
|
124
|
+
activity_1.log.debug(`Skipping ${type} embeddings for document ${document.id} - etag unchanged`);
|
|
125
|
+
return {
|
|
126
|
+
id: document.id,
|
|
127
|
+
type,
|
|
128
|
+
status: "skipped",
|
|
129
|
+
message: "embeddings already exist with matching etag",
|
|
130
|
+
};
|
|
131
|
+
}
|
|
132
|
+
// Count tokens if needed, do not rely on existing token count
|
|
133
|
+
let tokenCount = undefined;
|
|
134
|
+
if (type === common_1.SupportedEmbeddingTypes.text && document.text) {
|
|
135
|
+
tokenCount = (0, tokens_js_1.countTokens)(document.text).count;
|
|
136
|
+
}
|
|
137
|
+
const maxTokens = config.max_tokens ?? 8000;
|
|
138
|
+
//generate embeddings for the main doc if document isn't too large
|
|
139
|
+
activity_1.log.debug(`Generating ${type} embeddings for document ${document.id}`);
|
|
140
|
+
if (type === common_1.SupportedEmbeddingTypes.text &&
|
|
141
|
+
tokenCount !== undefined &&
|
|
142
|
+
tokenCount > maxTokens) {
|
|
143
|
+
//TODO: Review strategy for large documents
|
|
144
|
+
activity_1.log.warn(`Document too large for ${type} embeddings generation, skipping (${tokenCount} tokens)`);
|
|
145
|
+
return {
|
|
146
|
+
id: document.id,
|
|
147
|
+
status: "skipped",
|
|
148
|
+
message: `${type} embeddings generation, skipped for large document (${tokenCount} tokens)`,
|
|
149
|
+
};
|
|
150
|
+
}
|
|
151
|
+
else {
|
|
152
|
+
activity_1.log.debug(`Generating ${type} embeddings for document`);
|
|
153
|
+
const res = await generateEmbeddingsFromStudio(JSON.stringify(document[type]), environment, client);
|
|
154
|
+
if (!res || !res.values) {
|
|
155
|
+
return {
|
|
156
|
+
id: document.id,
|
|
157
|
+
status: "failed",
|
|
158
|
+
message: "no embeddings generated",
|
|
159
|
+
};
|
|
160
|
+
}
|
|
161
|
+
activity_1.log.debug(`${type} embeddings generated for document ${document.id}`, {
|
|
162
|
+
len: res.values.length,
|
|
163
|
+
});
|
|
164
|
+
await client.objects.setEmbedding(document.id, type, {
|
|
165
|
+
values: res.values,
|
|
166
|
+
model: res.model,
|
|
167
|
+
etag: textEtag,
|
|
168
|
+
});
|
|
169
|
+
return {
|
|
170
|
+
id: document.id,
|
|
171
|
+
type,
|
|
172
|
+
status: "completed",
|
|
173
|
+
len: res.values.length,
|
|
174
|
+
};
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
async function generateImageEmbeddings({ document, client, type, config, force, }) {
|
|
178
|
+
activity_1.log.debug("Generating image embeddings for document " + document.id, {
|
|
179
|
+
content: document.content,
|
|
180
|
+
});
|
|
181
|
+
if (!document.content?.type?.startsWith("image/") &&
|
|
182
|
+
!document.content?.type?.includes("pdf")) {
|
|
183
|
+
return {
|
|
184
|
+
id: document.id,
|
|
185
|
+
type,
|
|
186
|
+
status: "failed",
|
|
187
|
+
message: "content is not an image",
|
|
188
|
+
};
|
|
189
|
+
}
|
|
190
|
+
// Use content etag for image change detection
|
|
191
|
+
const contentEtag = document.content?.etag;
|
|
192
|
+
// Skip if embeddings already exist with matching etag (unless force=true)
|
|
193
|
+
const existingEmbedding = document.embeddings?.[type];
|
|
194
|
+
if (!force && existingEmbedding?.etag && contentEtag && existingEmbedding.etag === contentEtag) {
|
|
195
|
+
activity_1.log.debug(`Skipping ${type} embeddings for document ${document.id} - content etag unchanged`);
|
|
196
|
+
return {
|
|
197
|
+
id: document.id,
|
|
198
|
+
type,
|
|
199
|
+
status: "skipped",
|
|
200
|
+
message: "embeddings already exist with matching etag",
|
|
201
|
+
};
|
|
202
|
+
}
|
|
203
|
+
const { environment, model } = config;
|
|
204
|
+
const resRnd = await client.store.objects.getRendition(document.id, {
|
|
205
|
+
format: common_1.ImageRenditionFormat.jpeg,
|
|
206
|
+
generate_if_missing: true,
|
|
207
|
+
sign_url: true,
|
|
208
|
+
});
|
|
209
|
+
if (resRnd.status === "generating") {
|
|
210
|
+
throw new Error("Rendition is generating, will retry later");
|
|
211
|
+
}
|
|
212
|
+
else if (resRnd.status === "failed" ||
|
|
213
|
+
!resRnd.renditions ||
|
|
214
|
+
!resRnd.renditions.length) {
|
|
215
|
+
throw new errors_js_1.DocumentNotFoundError("Rendition retrieval failed", [document.id]);
|
|
216
|
+
}
|
|
217
|
+
const renditions = resRnd.renditions;
|
|
218
|
+
if (!renditions?.length) {
|
|
219
|
+
throw new errors_js_1.DocumentNotFoundError("No source found in rendition", [
|
|
220
|
+
document.id,
|
|
221
|
+
]);
|
|
222
|
+
}
|
|
223
|
+
const rendition = renditions[0];
|
|
224
|
+
const image = await (0, blobs_js_1.fetchBlobAsBase64)(client, rendition);
|
|
225
|
+
const res = await client.environments
|
|
226
|
+
.embeddings(environment, {
|
|
227
|
+
image,
|
|
228
|
+
model,
|
|
229
|
+
})
|
|
230
|
+
.then((res) => res)
|
|
231
|
+
.catch((e) => {
|
|
232
|
+
activity_1.log.error("Error generating embeddings for image", { error: e });
|
|
233
|
+
throw e;
|
|
234
|
+
});
|
|
235
|
+
if (!res || !res.values) {
|
|
236
|
+
return {
|
|
237
|
+
id: document.id,
|
|
238
|
+
status: "failed",
|
|
239
|
+
message: "no embeddings generated",
|
|
240
|
+
};
|
|
241
|
+
}
|
|
242
|
+
await client.objects.setEmbedding(document.id, common_1.SupportedEmbeddingTypes.image, {
|
|
243
|
+
values: res.values,
|
|
244
|
+
model: res.model,
|
|
245
|
+
etag: contentEtag,
|
|
246
|
+
});
|
|
247
|
+
return {
|
|
248
|
+
id: document.id,
|
|
249
|
+
type,
|
|
250
|
+
status: "completed",
|
|
251
|
+
len: res.values.length,
|
|
252
|
+
};
|
|
253
|
+
}
|
|
254
|
+
async function generateEmbeddingsFromStudio(text, env, client, model) {
|
|
255
|
+
activity_1.log.debug(`Generating embeddings for text of ${text.length} chars with environment ${env}`);
|
|
256
|
+
return client.environments
|
|
257
|
+
.embeddings(env, {
|
|
258
|
+
text,
|
|
259
|
+
model,
|
|
260
|
+
})
|
|
261
|
+
.then((res) => res)
|
|
262
|
+
.catch((e) => {
|
|
263
|
+
activity_1.log.error("Error generating embeddings for text", { error: e });
|
|
264
|
+
throw e;
|
|
265
|
+
});
|
|
266
|
+
}
|
|
267
|
+
//# sourceMappingURL=generateEmbeddings.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"generateEmbeddings.js","sourceRoot":"","sources":["../../../src/activities/generateEmbeddings.ts"],"names":[],"mappings":";;AAmDA,gDAyGC;AA3JD,mDAA2C;AAC3C,6CAA2E;AAC3E,6CAO0B;AAC1B,wEAAgE;AAChE,4CAAqD;AACrD,gDAA2D;AAE3D,kDAAiD;AAoC1C,KAAK,UAAU,kBAAkB,CACpC,OAA8D;IAE9D,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,YAAY,EAAE,GAC5C,MAAM,IAAA,kCAAa,EAA2B,OAAO,CAAC,CAAC;IAC3D,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,MAAM,CAAC;IAE/B,MAAM,WAAW,GAAG,MAAM,YAAY,EAAE,CAAC;IACzC,MAAM,MAAM,GAAG,WAAW,EAAE,aAAa,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;IAC3D,IAAI,CAAC,WAAW,EAAE,CAAC;QACf,MAAM,IAAI,iCAAqB,CAAC,mBAAmB,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;IAC/E,CAAC;IACD,IAAI,CAAC,MAAM,EAAE,CAAC;QACV,MAAM,IAAI,iCAAqB,CAAC,oCAAoC,EAAE;YAClE,QAAQ;SACX,CAAC,CAAC;IACP,CAAC;IAED,IAAI,CAAC,WAAW,EAAE,CAAC;QACf,MAAM,IAAI,iCAAqB,CAAC,mBAAmB,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;IAC/E,CAAC;IAED,IAAI,CAAC,WAAW,EAAE,aAAa,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,CAAC;QACxD,cAAG,CAAC,KAAK,CACL,2CAA2C,IAAI,gBAAgB,WAAW,CAAC,IAAI,KAAK,WAAW,CAAC,SAAS,GAAG,EAC5G,EAAE,MAAM,EAAE,CACb,CAAC;QACF,OAAO;YACH,EAAE,EAAE,QAAQ;YACZ,MAAM,EAAE,SAAS;YACjB,OAAO,EAAE,2CAA2C,IAAI,EAAE;SAC7D,CAAC;IACN,CAAC;IAED,cAAG,CAAC,KAAK,CAAC,GAAG,IAAI,6CAA6C,QAAQ,EAAE,EAAE;QACtE,KAAK;QACL,MAAM;KACT,CAAC,CAAC;IAEH,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QACtB,MAAM,IAAI,KAAK,CACX,iHAAiH,CACpH,CAAC;IACN,CAAC;IAED,IAAI,QAAQ,CAAC;IACb,IAAI,CAAC;QACD,QAAQ,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,QAAQ,CACpC,QAAQ,EACR,8CAA8C,CACjD,CAAC;IACN,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACb,IAAI,KAAK,YAAY,gCAAuB,EAAE,CAAC;YAC3C,MAAM,IAAI,iCAAqB,CAAC,uBAAuB,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC;QACnF,CAAC;QACD,MAAM,KAAK,CAAC;IAChB,CAAC;IAED,IAAI,CAAC,QAAQ,EAAE,CAAC;QACZ,MAAM,IAAI,iCAAqB,CAAC,oBAAoB,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC;IACtE,CAAC;IAED,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,IAAI,iCAAqB,CAAC,4BAA4B,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC9E,CAAC;IAED,IAAI,GAAG,CAAC;IAER,QAAQ,IAAI,EAAE,CAAC;QACX,KAAK,gCAAuB,CAAC,IAAI;YAC7B,GAAG,GAAG,MAAM,sBAAsB,CAAC;gBAC/B,MAAM;gBACN,MAAM;gBACN,QAAQ;gBACR,IAAI;gBACJ,KAAK;aACR,CAAC,CAAC;YACH,MAAM;QACV,KAAK,gCAAuB,CAAC,UAAU;YACnC,GAAG,GAAG,MAAM,sBAAsB,CAAC;gBAC/B,MAAM;gBACN,MAAM;gBACN,QAAQ;gBACR,IAAI;gBACJ,KAAK;aACR,CAAC,CAAC;YACH,MAAM;QACV,KAAK,gCAAuB,CAAC,KAAK;YAC9B,GAAG,GAAG,MAAM,uBAAuB,CAAC;gBAChC,MAAM;gBACN,MAAM;gBACN,QAAQ;gBACR,IAAI;gBACJ,KAAK;aACR,CAAC,CAAC;YACH,MAAM;QACV;YACI,GAAG,GAAG;gBACF,EAAE,EAAE,QAAQ;gBACZ,MAAM,EAAE,QAAQ;gBAChB,OAAO,EAAE,+BAA+B,IAAI,EAAE;aACjD,CAAC;IACV,CAAC;IAED,OAAO,GAAG,CAAC;AACf,CAAC;AAWD,KAAK,UAAU,sBAAsB,CACjC,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAmC;IAG1E,IAAI,CAAC,QAAQ,EAAE,CAAC;QACZ,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,+BAA+B,EAAE,CAAC;IACzE,CAAC;IAED,IACI,IAAI,KAAK,gCAAuB,CAAC,IAAI;QACrC,IAAI,KAAK,gCAAuB,CAAC,UAAU,EAC7C,CAAC;QACC,OAAO;YACH,EAAE,EAAE,QAAQ,CAAC,EAAE;YACf,MAAM,EAAE,QAAQ;YAChB,OAAO,EAAE,+BAA+B,IAAI,EAAE;SACjD,CAAC;IACN,CAAC;IAED,IAAI,IAAI,KAAK,gCAAuB,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QAC1D,OAAO,EAAE,EAAE,EAAE,QAAQ,CAAC,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,eAAe,EAAE,CAAC;IAC3E,CAAC;IACD,IAAI,IAAI,KAAK,gCAAuB,CAAC,UAAU,IAAI,CAAC,QAAQ,EAAE,UAAU,EAAE,CAAC;QACvE,OAAO;YACH,EAAE,EAAE,QAAQ,CAAC,EAAE;YACf,MAAM,EAAE,QAAQ;YAChB,OAAO,EAAE,qBAAqB;SACjC,CAAC;IACN,CAAC;IAED,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,CAAC;IAE/B,mCAAmC;IACnC,MAAM,QAAQ,GAAG,QAAQ,CAAC,SAAS,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,IAAA,cAAG,EAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IAExF,0EAA0E;IAC1E,MAAM,iBAAiB,GAAG,QAAQ,CAAC,UAAU,EAAE,CAAC,IAAI,CAAC,CAAC;IACtD,IAAI,CAAC,KAAK,IAAI,iBAAiB,EAAE,IAAI,IAAI,QAAQ,IAAI,iBAAiB,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QACvF,cAAG,CAAC,KAAK,CAAC,YAAY,IAAI,4BAA4B,QAAQ,CAAC,EAAE,mBAAmB,CAAC,CAAC;QACtF,OAAO;YACH,EAAE,EAAE,QAAQ,CAAC,EAAE;YACf,IAAI;YACJ,MAAM,EAAE,SAAS;YACjB,OAAO,EAAE,6CAA6C;SACzD,CAAC;IACN,CAAC;IAED,8DAA8D;IAC9D,IAAI,UAAU,GAAwB,SAAS,CAAC;IAChD,IAAI,IAAI,KAAK,gCAAuB,CAAC,IAAI,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC;QACzD,UAAU,GAAG,IAAA,uBAAW,EAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC;IAClD,CAAC;IAED,MAAM,SAAS,GAAG,MAAM,CAAC,UAAU,IAAI,IAAI,CAAC;IAE5C,kEAAkE;IAClE,cAAG,CAAC,KAAK,CAAC,cAAc,IAAI,4BAA4B,QAAQ,CAAC,EAAE,EAAE,CAAC,CAAC;IACvE,IACI,IAAI,KAAK,gCAAuB,CAAC,IAAI;QACrC,UAAU,KAAK,SAAS;QACxB,UAAU,GAAG,SAAS,EACxB,CAAC;QACC,2CAA2C;QAC3C,cAAG,CAAC,IAAI,CACJ,0BAA0B,IAAI,qCAAqC,UAAU,UAAU,CAC1F,CAAC;QACF,OAAO;YACH,EAAE,EAAE,QAAQ,CAAC,EAAE;YACf,MAAM,EAAE,SAAS;YACjB,OAAO,EAAE,GAAG,IAAI,uDAAuD,UAAU,UAAU;SAC9F,CAAA;IACL,CAAC;SAAM,CAAC;QACJ,cAAG,CAAC,KAAK,CAAC,cAAc,IAAI,0BAA0B,CAAC,CAAC;QAExD,MAAM,GAAG,GAAG,MAAM,4BAA4B,CAC1C,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,EAC9B,WAAW,EACX,MAAM,CACT,CAAC;QACF,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC;YACtB,OAAO;gBACH,EAAE,EAAE,QAAQ,CAAC,EAAE;gBACf,MAAM,EAAE,QAAQ;gBAChB,OAAO,EAAE,yBAAyB;aACrC,CAAC;QACN,CAAC;QAED,cAAG,CAAC,KAAK,CAAC,GAAG,IAAI,sCAAsC,QAAQ,CAAC,EAAE,EAAE,EAAE;YAClE,GAAG,EAAE,GAAG,CAAC,MAAM,CAAC,MAAM;SACzB,CAAC,CAAC;QACH,MAAM,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,QAAQ,CAAC,EAAE,EAAE,IAAI,EAAE;YACjD,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,KAAK,EAAE,GAAG,CAAC,KAAK;YAChB,IAAI,EAAE,QAAQ;SACjB,CAAC,CAAC;QAEH,OAAO;YACH,EAAE,EAAE,QAAQ,CAAC,EAAE;YACf,IAAI;YACJ,MAAM,EAAE,WAAW;YACnB,GAAG,EAAE,GAAG,CAAC,MAAM,CAAC,MAAM;SACzB,CAAC;IACN,CAAC;AACL,CAAC;AAED,KAAK,UAAU,uBAAuB,CAAC,EACnC,QAAQ,EACR,MAAM,EACN,IAAI,EACJ,MAAM,EACN,KAAK,GACyB;IAC9B,cAAG,CAAC,KAAK,CAAC,2CAA2C,GAAG,QAAQ,CAAC,EAAE,EAAE;QACjE,OAAO,EAAE,QAAQ,CAAC,OAAO;KAC5B,CAAC,CAAC;IACH,IACI,CAAC,QAAQ,CAAC,OAAO,EAAE,IAAI,EAAE,UAAU,CAAC,QAAQ,CAAC;QAC7C,CAAC,QAAQ,CAAC,OAAO,EAAE,IAAI,EAAE,QAAQ,CAAC,KAAK,CAAC,EAC1C,CAAC;QACC,OAAO;YACH,EAAE,EAAE,QAAQ,CAAC,EAAE;YACf,IAAI;YACJ,MAAM,EAAE,QAAQ;YAChB,OAAO,EAAE,yBAAyB;SACrC,CAAC;IACN,CAAC;IAED,8CAA8C;IAC9C,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,EAAE,IAAI,CAAC;IAE3C,0EAA0E;IAC1E,MAAM,iBAAiB,GAAG,QAAQ,CAAC,UAAU,EAAE,CAAC,IAAI,CAAC,CAAC;IACtD,IAAI,CAAC,KAAK,IAAI,iBAAiB,EAAE,IAAI,IAAI,WAAW,IAAI,iBAAiB,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;QAC7F,cAAG,CAAC,KAAK,CAAC,YAAY,IAAI,4BAA4B,QAAQ,CAAC,EAAE,2BAA2B,CAAC,CAAC;QAC9F,OAAO;YACH,EAAE,EAAE,QAAQ,CAAC,EAAE;YACf,IAAI;YACJ,MAAM,EAAE,SAAS;YACjB,OAAO,EAAE,6CAA6C;SACzD,CAAC;IACN,CAAC;IAED,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC;IAEtC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,YAAY,CAAC,QAAQ,CAAC,EAAE,EAAE;QAChE,MAAM,EAAE,6BAAoB,CAAC,IAAI;QACjC,mBAAmB,EAAE,IAAI;QACzB,QAAQ,EAAE,IAAI;KACjB,CAAC,CAAC;IAEH,IAAI,MAAM,CAAC,MAAM,KAAK,YAAY,EAAE,CAAC;QACjC,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;IACjE,CAAC;SAAM,IACH,MAAM,CAAC,MAAM,KAAK,QAAQ;QAC1B,CAAC,MAAM,CAAC,UAAU;QAClB,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,EAC3B,CAAC;QACC,MAAM,IAAI,iCAAqB,CAAC,4BAA4B,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;IACjF,CAAC;IAED,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC;IACrC,IAAI,CAAC,UAAU,EAAE,MAAM,EAAE,CAAC;QACtB,MAAM,IAAI,iCAAqB,CAAC,8BAA8B,EAAE;YAC5D,QAAQ,CAAC,EAAE;SACd,CAAC,CAAC;IACP,CAAC;IAED,MAAM,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;IAChC,MAAM,KAAK,GAAG,MAAM,IAAA,4BAAiB,EAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IAEzD,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,YAAY;SAChC,UAAU,CAAC,WAAW,EAAE;QACrB,KAAK;QACL,KAAK;KACR,CAAC;SACD,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC;SAClB,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;QACT,cAAG,CAAC,KAAK,CAAC,uCAAuC,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;QACjE,MAAM,CAAC,CAAC;IACZ,CAAC,CAAC,CAAC;IAEP,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC;QACtB,OAAO;YACH,EAAE,EAAE,QAAQ,CAAC,EAAE;YACf,MAAM,EAAE,QAAQ;YAChB,OAAO,EAAE,yBAAyB;SACrC,CAAC;IACN,CAAC;IAED,MAAM,MAAM,CAAC,OAAO,CAAC,YAAY,CAC7B,QAAQ,CAAC,EAAE,EACX,gCAAuB,CAAC,KAAK,EAC7B;QACI,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,IAAI,EAAE,WAAW;KACpB,CACJ,CAAC;IAEF,OAAO;QACH,EAAE,EAAE,QAAQ,CAAC,EAAE;QACf,IAAI;QACJ,MAAM,EAAE,WAAW;QACnB,GAAG,EAAE,GAAG,CAAC,MAAM,CAAC,MAAM;KACzB,CAAC;AACN,CAAC;AAED,KAAK,UAAU,4BAA4B,CACvC,IAAY,EACZ,GAAW,EACX,MAAsB,EACtB,KAAc;IAEd,cAAG,CAAC,KAAK,CACL,qCAAqC,IAAI,CAAC,MAAM,2BAA2B,GAAG,EAAE,CACnF,CAAC;IAEF,OAAO,MAAM,CAAC,YAAY;SACrB,UAAU,CAAC,GAAG,EAAE;QACb,IAAI;QACJ,KAAK;KACR,CAAC;SACD,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC;SAClB,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;QACT,cAAG,CAAC,KAAK,CAAC,sCAAsC,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;QAChE,MAAM,CAAC,CAAC;IACZ,CAAC,CAAC,CAAC;AACX,CAAC"}
|