@vertesia/workflow 0.51.0 → 0.52.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (141) hide show
  1. package/lib/cjs/activities/advanced/createOrUpdateDocumentFromInteractionRun.js +7 -1
  2. package/lib/cjs/activities/advanced/createOrUpdateDocumentFromInteractionRun.js.map +1 -1
  3. package/lib/cjs/activities/chunkDocument.js +39 -34
  4. package/lib/cjs/activities/chunkDocument.js.map +1 -1
  5. package/lib/cjs/activities/createDocumentFromOther.js +2 -2
  6. package/lib/cjs/activities/createDocumentFromOther.js.map +1 -1
  7. package/lib/cjs/activities/executeInteraction.js +11 -5
  8. package/lib/cjs/activities/executeInteraction.js.map +1 -1
  9. package/lib/cjs/activities/extractDocumentText.js +24 -6
  10. package/lib/cjs/activities/extractDocumentText.js.map +1 -1
  11. package/lib/cjs/activities/generateDocumentProperties.js +22 -4
  12. package/lib/cjs/activities/generateDocumentProperties.js.map +1 -1
  13. package/lib/cjs/activities/generateEmbeddings.js +58 -102
  14. package/lib/cjs/activities/generateEmbeddings.js.map +1 -1
  15. package/lib/cjs/activities/generateImageRendition.js +77 -34
  16. package/lib/cjs/activities/generateImageRendition.js.map +1 -1
  17. package/lib/cjs/activities/generateOrAssignContentType.js +3 -7
  18. package/lib/cjs/activities/generateOrAssignContentType.js.map +1 -1
  19. package/lib/cjs/activities/notifyWebhook.js.map +1 -1
  20. package/lib/cjs/conversion/image.js +80 -12
  21. package/lib/cjs/conversion/image.js.map +1 -1
  22. package/lib/cjs/dsl/setup/ActivityContext.js +30 -6
  23. package/lib/cjs/dsl/setup/ActivityContext.js.map +1 -1
  24. package/lib/cjs/dsl.js +1 -1
  25. package/lib/cjs/dsl.js.map +1 -1
  26. package/lib/cjs/errors.js +13 -1
  27. package/lib/cjs/errors.js.map +1 -1
  28. package/lib/cjs/iterative-generation/iterativeGenerationWorkflow.js +2 -1
  29. package/lib/cjs/iterative-generation/iterativeGenerationWorkflow.js.map +1 -1
  30. package/lib/cjs/system/notifyWebhookWorkflow.js +2 -1
  31. package/lib/cjs/system/notifyWebhookWorkflow.js.map +1 -1
  32. package/lib/cjs/system/recalculateEmbeddingsWorkflow.js +1 -1
  33. package/lib/cjs/system/recalculateEmbeddingsWorkflow.js.map +1 -1
  34. package/lib/cjs/utils/blobs.js +12 -6
  35. package/lib/cjs/utils/blobs.js.map +1 -1
  36. package/lib/cjs/utils/chunks.js +14 -0
  37. package/lib/cjs/utils/chunks.js.map +1 -0
  38. package/lib/cjs/utils/client.js +4 -3
  39. package/lib/cjs/utils/client.js.map +1 -1
  40. package/lib/cjs/utils/memory.js +2 -9
  41. package/lib/cjs/utils/memory.js.map +1 -1
  42. package/lib/esm/activities/advanced/createOrUpdateDocumentFromInteractionRun.js +7 -1
  43. package/lib/esm/activities/advanced/createOrUpdateDocumentFromInteractionRun.js.map +1 -1
  44. package/lib/esm/activities/chunkDocument.js +39 -34
  45. package/lib/esm/activities/chunkDocument.js.map +1 -1
  46. package/lib/esm/activities/createDocumentFromOther.js +1 -1
  47. package/lib/esm/activities/createDocumentFromOther.js.map +1 -1
  48. package/lib/esm/activities/executeInteraction.js +11 -5
  49. package/lib/esm/activities/executeInteraction.js.map +1 -1
  50. package/lib/esm/activities/extractDocumentText.js +24 -6
  51. package/lib/esm/activities/extractDocumentText.js.map +1 -1
  52. package/lib/esm/activities/generateDocumentProperties.js +22 -4
  53. package/lib/esm/activities/generateDocumentProperties.js.map +1 -1
  54. package/lib/esm/activities/generateEmbeddings.js +58 -69
  55. package/lib/esm/activities/generateEmbeddings.js.map +1 -1
  56. package/lib/esm/activities/generateImageRendition.js +78 -35
  57. package/lib/esm/activities/generateImageRendition.js.map +1 -1
  58. package/lib/esm/activities/generateOrAssignContentType.js +3 -7
  59. package/lib/esm/activities/generateOrAssignContentType.js.map +1 -1
  60. package/lib/esm/activities/notifyWebhook.js.map +1 -1
  61. package/lib/esm/conversion/image.js +80 -12
  62. package/lib/esm/conversion/image.js.map +1 -1
  63. package/lib/esm/dsl/setup/ActivityContext.js +31 -7
  64. package/lib/esm/dsl/setup/ActivityContext.js.map +1 -1
  65. package/lib/esm/dsl.js +1 -1
  66. package/lib/esm/dsl.js.map +1 -1
  67. package/lib/esm/errors.js +11 -0
  68. package/lib/esm/errors.js.map +1 -1
  69. package/lib/esm/iterative-generation/iterativeGenerationWorkflow.js +2 -1
  70. package/lib/esm/iterative-generation/iterativeGenerationWorkflow.js.map +1 -1
  71. package/lib/esm/system/notifyWebhookWorkflow.js +2 -1
  72. package/lib/esm/system/notifyWebhookWorkflow.js.map +1 -1
  73. package/lib/esm/system/recalculateEmbeddingsWorkflow.js +2 -2
  74. package/lib/esm/system/recalculateEmbeddingsWorkflow.js.map +1 -1
  75. package/lib/esm/utils/blobs.js +12 -6
  76. package/lib/esm/utils/blobs.js.map +1 -1
  77. package/lib/esm/utils/chunks.js +9 -0
  78. package/lib/esm/utils/chunks.js.map +1 -0
  79. package/lib/esm/utils/client.js +4 -3
  80. package/lib/esm/utils/client.js.map +1 -1
  81. package/lib/esm/utils/memory.js +2 -7
  82. package/lib/esm/utils/memory.js.map +1 -1
  83. package/lib/types/activities/advanced/createOrUpdateDocumentFromInteractionRun.d.ts +10 -0
  84. package/lib/types/activities/advanced/createOrUpdateDocumentFromInteractionRun.d.ts.map +1 -1
  85. package/lib/types/activities/chunkDocument.d.ts +15 -0
  86. package/lib/types/activities/chunkDocument.d.ts.map +1 -1
  87. package/lib/types/activities/createDocumentFromOther.d.ts.map +1 -1
  88. package/lib/types/activities/executeInteraction.d.ts +14 -3
  89. package/lib/types/activities/executeInteraction.d.ts.map +1 -1
  90. package/lib/types/activities/generateDocumentProperties.d.ts +1 -1
  91. package/lib/types/activities/generateDocumentProperties.d.ts.map +1 -1
  92. package/lib/types/activities/generateEmbeddings.d.ts +21 -17
  93. package/lib/types/activities/generateEmbeddings.d.ts.map +1 -1
  94. package/lib/types/activities/generateImageRendition.d.ts +3 -5
  95. package/lib/types/activities/generateImageRendition.d.ts.map +1 -1
  96. package/lib/types/activities/generateOrAssignContentType.d.ts.map +1 -1
  97. package/lib/types/activities/notifyWebhook.d.ts +1 -2
  98. package/lib/types/activities/notifyWebhook.d.ts.map +1 -1
  99. package/lib/types/conversion/image.d.ts +8 -6
  100. package/lib/types/conversion/image.d.ts.map +1 -1
  101. package/lib/types/dsl/setup/ActivityContext.d.ts +3 -0
  102. package/lib/types/dsl/setup/ActivityContext.d.ts.map +1 -1
  103. package/lib/types/dsl.d.ts +1 -1
  104. package/lib/types/dsl.d.ts.map +1 -1
  105. package/lib/types/errors.d.ts +6 -0
  106. package/lib/types/errors.d.ts.map +1 -1
  107. package/lib/types/iterative-generation/iterativeGenerationWorkflow.d.ts.map +1 -1
  108. package/lib/types/system/notifyWebhookWorkflow.d.ts.map +1 -1
  109. package/lib/types/system/recalculateEmbeddingsWorkflow.d.ts +2 -17
  110. package/lib/types/system/recalculateEmbeddingsWorkflow.d.ts.map +1 -1
  111. package/lib/types/utils/blobs.d.ts.map +1 -1
  112. package/lib/types/utils/chunks.d.ts +9 -0
  113. package/lib/types/utils/chunks.d.ts.map +1 -0
  114. package/lib/types/utils/client.d.ts.map +1 -1
  115. package/lib/types/utils/memory.d.ts +1 -5
  116. package/lib/types/utils/memory.d.ts.map +1 -1
  117. package/lib/workflows-bundle.js +15394 -14602
  118. package/package.json +8 -6
  119. package/src/activities/advanced/createOrUpdateDocumentFromInteractionRun.ts +20 -1
  120. package/src/activities/chunkDocument.ts +62 -42
  121. package/src/activities/createDocumentFromOther.ts +1 -1
  122. package/src/activities/executeInteraction.ts +27 -9
  123. package/src/activities/extractDocumentText.ts +28 -7
  124. package/src/activities/generateDocumentProperties.ts +37 -16
  125. package/src/activities/generateEmbeddings.ts +91 -79
  126. package/src/activities/generateImageRendition.ts +100 -53
  127. package/src/activities/generateOrAssignContentType.ts +5 -11
  128. package/src/activities/notifyWebhook.ts +2 -2
  129. package/src/conversion/image.test.ts +110 -18
  130. package/src/conversion/image.ts +90 -15
  131. package/src/conversion/pandoc.test.ts +7 -5
  132. package/src/dsl/setup/ActivityContext.ts +57 -16
  133. package/src/dsl.ts +1 -1
  134. package/src/errors.ts +27 -6
  135. package/src/iterative-generation/iterativeGenerationWorkflow.ts +2 -1
  136. package/src/system/notifyWebhookWorkflow.ts +2 -1
  137. package/src/system/recalculateEmbeddingsWorkflow.ts +2 -2
  138. package/src/utils/blobs.ts +11 -6
  139. package/src/utils/chunks.ts +17 -0
  140. package/src/utils/client.ts +4 -3
  141. package/src/utils/memory.ts +3 -8
@@ -1,8 +1,8 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.createOrUpdateDocumentFromInteractionRun = createOrUpdateDocumentFromInteractionRun;
4
- const common_1 = require("@vertesia/common");
5
4
  const activity_1 = require("@temporalio/activity");
5
+ const common_1 = require("@vertesia/common");
6
6
  const ActivityContext_js_1 = require("../../dsl/setup/ActivityContext.js");
7
7
  const errors_js_1 = require("../../errors.js");
8
8
  async function createOrUpdateDocumentFromInteractionRun(payload) {
@@ -48,6 +48,12 @@ async function createOrUpdateDocumentFromInteractionRun(payload) {
48
48
  target: resultIsObject ? 'properties' : 'text'
49
49
  }
50
50
  };
51
+ if (params.update_text_from_property) {
52
+ const text = docPayload.properties[params.update_text_from_property];
53
+ if (text) {
54
+ docPayload.text = text;
55
+ }
56
+ }
51
57
  //create or update the document
52
58
  let newDoc = false;
53
59
  let doc = undefined;
@@ -1 +1 @@
1
- {"version":3,"file":"createOrUpdateDocumentFromInteractionRun.js","sourceRoot":"","sources":["../../../../src/activities/advanced/createOrUpdateDocumentFromInteractionRun.ts"],"names":[],"mappings":";;AA8BA,4FAkEC;AAhGD,6CAAqG;AACrG,mDAA2C;AAC3C,2EAAmE;AACnE,+CAAyE;AA2BlE,KAAK,UAAU,wCAAwC,CAAC,OAAkF;IAC7I,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,IAAA,kCAAa,EAA+C,OAAO,CAAC,CAAC;IAEtG,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC;IAC5B,MAAM,cAAc,GAAG,MAAM,CAAC,WAAW,CAAC;IAE1C,IAAI,CAAC,KAAK,EAAE,CAAC;QACT,MAAM,IAAI,iCAAqB,CAAC,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IAChE,CAAC;IACD,IAAI,CAAC,cAAc,IAAI,CAAC,MAAM,CAAC,kBAAkB,EAAE,CAAC;QAChD,MAAM,IAAI,iCAAqB,CAAC,aAAa,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IACrE,CAAC;IAED,cAAG,CAAC,IAAI,CAAC,2CAA2C,EAAE,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,CAAC;IAEjF,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;QACtD,MAAM,IAAI,2BAAe,CAAC,sBAAsB,KAAK,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;IAC3E,CAAC,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,cAAc,CAAC,CAAC;QACzB,MAAM,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,cAAc,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;YACzD,MAAM,IAAI,2BAAe,CAAC,uBAAuB,cAAc,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;QACrF,CAAC,CAAC;QACF,CAAC,CAAC,SAAS,CAAC;IAGhB,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC;IAC1B,MAAM,cAAc,GAAG,OAAO,MAAM,KAAK,QAAQ,CAAC;IAClD,MAAM,SAAS,GAAG,GAAG,CAAC,UAAU,CAAC;IAEjC,IAAI,IAAY,CAAC;IACjB,IAAI,cAAc,EAAE,CAAC;QACjB,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,SAAS,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,aAAa,IAAI,UAAU,CAAC;IACxG,CAAC;SAAM,CAAC;QACJ,IAAI,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,aAAa,IAAI,UAAU,CAAC;IACnE,CAAC;IAED,MAAM,UAAU,GAAG;QACf,IAAI;QACJ,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,SAAS;QAClC,UAAU,EAAE,cAAc,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE;QACxC,IAAI,EAAE,CAAC,cAAc,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;QAC1C,IAAI,EAAE,IAAI,EAAE,EAAE;QACd,MAAM,EAAE,4BAAmB,CAAC,SAAS;QACrC,mBAAmB,EAAE;YACjB,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YAC9B,KAAK,EAAE,GAAG,CAAC,OAAO;YAClB,MAAM,EAAE,cAAc,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM;SACjD;KACJ,CAAC;IAEF,+BAA+B;IAC/B,IAAI,MAAM,GAAY,KAAK,CAAC;IAC5B,IAAI,GAAG,GAAG,SAAS,CAAC;IACpB,IAAI,MAAM,CAAC,kBAAkB,EAAE,CAAC;QAC5B,cAAG,CAAC,IAAI,CAAC,8BAA8B,MAAM,CAAC,kBAAkB,EAAE,CAAC,CAAC;QACpE,GAAG,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,kBAAkB,EAAE,UAAU,CAAC,CAAC;IAC7E,CAAC;SAAM,CAAC;QACJ,cAAG,CAAC,IAAI,CAAC,iCAAiC,cAAc,EAAE,CAAC,CAAC;QAC5D,GAAG,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAC9C,MAAM,GAAG,IAAI,CAAC;IAClB,CAAC;IAED,cAAG,CAAC,IAAI,CAAC,YAAY,cAAc,GAAG,GAAG,GAAG,GAAG,CAAC,EAAE,IAAI,GAAG,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC;IACrG,OAAO,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAA;AACpD,CAAC"}
1
+ {"version":3,"file":"createOrUpdateDocumentFromInteractionRun.js","sourceRoot":"","sources":["../../../../src/activities/advanced/createOrUpdateDocumentFromInteractionRun.ts"],"names":[],"mappings":";;AA0CA,4FAyEC;AAnHD,mDAA2C;AAC3C,6CAAqG;AACrG,2EAAmE;AACnE,+CAAyE;AAuClE,KAAK,UAAU,wCAAwC,CAAC,OAAkF;IAC7I,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,IAAA,kCAAa,EAA+C,OAAO,CAAC,CAAC;IAEtG,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC;IAC5B,MAAM,cAAc,GAAG,MAAM,CAAC,WAAW,CAAC;IAE1C,IAAI,CAAC,KAAK,EAAE,CAAC;QACT,MAAM,IAAI,iCAAqB,CAAC,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IAChE,CAAC;IACD,IAAI,CAAC,cAAc,IAAI,CAAC,MAAM,CAAC,kBAAkB,EAAE,CAAC;QAChD,MAAM,IAAI,iCAAqB,CAAC,aAAa,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IACrE,CAAC;IAED,cAAG,CAAC,IAAI,CAAC,2CAA2C,EAAE,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,CAAC;IAEjF,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;QACtD,MAAM,IAAI,2BAAe,CAAC,sBAAsB,KAAK,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;IAC3E,CAAC,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,cAAc,CAAC,CAAC;QACzB,MAAM,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,cAAc,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;YACzD,MAAM,IAAI,2BAAe,CAAC,uBAAuB,cAAc,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;QACrF,CAAC,CAAC;QACF,CAAC,CAAC,SAAS,CAAC;IAGhB,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC;IAC1B,MAAM,cAAc,GAAG,OAAO,MAAM,KAAK,QAAQ,CAAC;IAClD,MAAM,SAAS,GAAG,GAAG,CAAC,UAAU,CAAC;IAEjC,IAAI,IAAY,CAAC;IACjB,IAAI,cAAc,EAAE,CAAC;QACjB,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,SAAS,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,aAAa,IAAI,UAAU,CAAC;IACxG,CAAC;SAAM,CAAC;QACJ,IAAI,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,aAAa,IAAI,UAAU,CAAC;IACnE,CAAC;IAED,MAAM,UAAU,GAAG;QACf,IAAI;QACJ,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,SAAS;QAClC,UAAU,EAAE,cAAc,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE;QACxC,IAAI,EAAE,CAAC,cAAc,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;QAC1C,IAAI,EAAE,IAAI,EAAE,EAAE;QACd,MAAM,EAAE,4BAAmB,CAAC,SAAS;QACrC,mBAAmB,EAAE;YACjB,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YAC9B,KAAK,EAAE,GAAG,CAAC,OAAO;YAClB,MAAM,EAAE,cAAc,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM;SACjD;KACJ,CAAC;IAEF,IAAI,MAAM,CAAC,yBAAyB,EAAE,CAAC;QACnC,MAAM,IAAI,GAAG,UAAU,CAAC,UAAU,CAAC,MAAM,CAAC,yBAAyB,CAAC,CAAC;QACrE,IAAI,IAAI,EAAE,CAAC;YACP,UAAU,CAAC,IAAI,GAAG,IAAI,CAAC;QAC3B,CAAC;IACL,CAAC;IAED,+BAA+B;IAC/B,IAAI,MAAM,GAAY,KAAK,CAAC;IAC5B,IAAI,GAAG,GAAG,SAAS,CAAC;IACpB,IAAI,MAAM,CAAC,kBAAkB,EAAE,CAAC;QAC5B,cAAG,CAAC,IAAI,CAAC,8BAA8B,MAAM,CAAC,kBAAkB,EAAE,CAAC,CAAC;QACpE,GAAG,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,kBAAkB,EAAE,UAAU,CAAC,CAAC;IAC7E,CAAC;SAAM,CAAC;QACJ,cAAG,CAAC,IAAI,CAAC,iCAAiC,cAAc,EAAE,CAAC,CAAC;QAC5D,GAAG,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAC9C,MAAM,GAAG,IAAI,CAAC;IAClB,CAAC;IAED,cAAG,CAAC,IAAI,CAAC,YAAY,cAAc,GAAG,GAAG,GAAG,GAAG,CAAC,EAAE,IAAI,GAAG,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC;IACrG,OAAO,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAA;AACpD,CAAC"}
@@ -37,42 +37,47 @@ async function chunkDocument(payload) {
37
37
  activity_1.log.warn('No parts found for object ID: ' + objectId, res);
38
38
  return { id: objectId, status: "failed", parts: [], message: "no parts found" };
39
39
  }
40
- const partDocs = await Promise.all(parts.map(async (part, i) => {
41
- const text = lines.filter((_l, i) => i >= part.line_number_start && i <= part.line_number_end).join('\n');
42
- const location = () => {
43
- let location = document.location;
44
- if (location.endsWith('/')) {
45
- location += document.name + "/" + part.type;
46
- }
47
- location += '/' + document.name + "/" + part.type;
48
- return location;
49
- };
50
- const docPart = await client.objects.create({
51
- name: part.name,
52
- parent: objectId,
53
- text: text,
54
- location: location(),
55
- properties: {
56
- part_number: i + 1,
57
- etag: document.text_etag,
58
- source_line_start: part.line_number_start,
59
- source_line_end: part.line_number_end,
60
- title: part.name
61
- }
62
- });
63
- return docPart;
64
- }));
65
- //delete previous parts
66
- if (document.parts && document.parts.length > 0) {
67
- activity_1.log.info('Deleting previous parts for object ID: ' + objectId, { parts: document.parts });
68
- await Promise.all(document.parts.map(async (partId) => {
69
- await client.objects.delete(partId);
40
+ /**
41
+ * Only create parts as document if the flag is set
42
+ */
43
+ if (params.createParts) {
44
+ const partDocs = await Promise.all(parts.map(async (part, i) => {
45
+ const text = lines.filter((_l, i) => i >= part.line_number_start && i <= part.line_number_end).join('\n');
46
+ const location = () => {
47
+ let location = document.location;
48
+ if (location.endsWith('/')) {
49
+ location += document.name + "/" + part.type;
50
+ }
51
+ location += '/' + document.name + "/" + part.type;
52
+ return location;
53
+ };
54
+ const docPart = await client.objects.create({
55
+ name: part.name,
56
+ parent: objectId,
57
+ text: text,
58
+ location: location(),
59
+ properties: {
60
+ part_number: i + 1,
61
+ etag: document.text_etag,
62
+ source_line_start: part.line_number_start,
63
+ source_line_end: part.line_number_end,
64
+ title: part.name
65
+ }
66
+ });
67
+ return docPart;
70
68
  }));
69
+ //delete previous parts
70
+ if (document.parts && document.parts.length > 0) {
71
+ activity_1.log.info('Deleting previous parts for object ID: ' + objectId, { parts: document.parts });
72
+ await Promise.all(document.parts.map(async (partId) => {
73
+ await client.objects.delete(partId);
74
+ }));
75
+ }
76
+ await client.objects.update(objectId, {
77
+ parts: partDocs.map(p => p.id),
78
+ parts_etag: document.text_etag
79
+ });
71
80
  }
72
- await client.objects.update(objectId, {
73
- parts: partDocs.map(p => p.id),
74
- parts_etag: document.text_etag
75
- });
76
81
  activity_1.log.info(`Object ${objectId} chunking completed`, { parts: document.parts });
77
82
  return { id: objectId, status: "completed", parts: document.parts };
78
83
  }
@@ -1 +1 @@
1
- {"version":3,"file":"chunkDocument.js","sourceRoot":"","sources":["../../../src/activities/chunkDocument.ts"],"names":[],"mappings":";;AAkCA,sCAyFC;AA1HD,mDAA2C;AAC3C,wEAAgE;AAChE,mEAAqG;AAErG,MAAM,kBAAkB,GAAG,mBAAmB,CAAA;AA6BvC,KAAK,UAAU,aAAa,CAAC,OAAyD;IACzF,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAA,kCAAa,EAAsB,OAAO,CAAC,CAAC;IAEvF,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC;IACzB,MAAM,eAAe,GAAG,MAAM,CAAC,eAAe,IAAI,kBAAkB,CAAC;IAErE,cAAG,CAAC,IAAI,CAAC,UAAU,QAAQ,mBAAmB,CAAC,CAAC;IAEhD,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAElE,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAEvF,IAAI,CAAC,IAAI,EAAE,YAAY,EAAE,CAAC;QACtB,cAAG,CAAC,IAAI,CAAC,uCAAuC,GAAG,QAAQ,CAAC,CAAC;QAC7D,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,oBAAoB,EAAE,CAAA;IAC7E,CAAC;IAED,0BAA0B;IAC1B,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QACjB,cAAG,CAAC,IAAI,CAAC,+BAA+B,GAAG,QAAQ,CAAC,CAAC;QACrD,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,eAAe,EAAE,CAAA;IACvE,CAAC;IAED,IAAI,CAAC,KAAK,IAAI,QAAQ,CAAC,KAAK,IAAI,QAAQ,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,QAAQ,CAAC,UAAU,KAAK,QAAQ,CAAC,SAAS,EAAE,CAAC;QACtG,cAAG,CAAC,IAAI,CAAC,0CAA0C,GAAG,QAAQ,CAAC,CAAC;QAChE,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,4CAA4C,EAAE,CAAA;IACrG,CAAC;IAED,uCAAuC;IACvC,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IACvC,MAAM,YAAY,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAEnE,MAAM,GAAG,GAAG,MAAM,IAAA,sDAA8B,EAAC,MAAM,EAAE,eAAe,EAAE,MAAM,EAAE;QAC9E,QAAQ,EAAE,QAAQ;QAClB,OAAO,EAAE,YAAY;KACxB,CAAC,CAAC;IAEH,MAAM,KAAK,GAAG,GAAG,CAAC,MAAM,CAAC,KAAkB,CAAC;IAC5C,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/B,cAAG,CAAC,IAAI,CAAC,gCAAgC,GAAG,QAAQ,EAAE,GAAG,CAAC,CAAC;QAC3D,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,EAAE,EAAE,OAAO,EAAE,gBAAgB,EAAE,CAAA;IACnF,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE;QAE3D,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,IAAI,CAAC,iBAAiB,IAAI,CAAC,IAAI,IAAI,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE1G,MAAM,QAAQ,GAAG,GAAG,EAAE;YAClB,IAAI,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC;YACjC,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gBACzB,QAAQ,IAAI,QAAQ,CAAC,IAAI,GAAG,GAAG,GAAG,IAAI,CAAC,IAAI,CAAA;YAC/C,CAAC;YACD,QAAQ,IAAI,GAAG,GAAG,QAAQ,CAAC,IAAI,GAAG,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC;YAClD,OAAO,QAAQ,CAAC;QACpB,CAAC,CAAA;QAED,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC;YACxC,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,MAAM,EAAE,QAAQ;YAChB,IAAI,EAAE,IAAI;YACV,QAAQ,EAAE,QAAQ,EAAE;YACpB,UAAU,EAAE;gBACR,WAAW,EAAE,CAAC,GAAG,CAAC;gBAClB,IAAI,EAAE,QAAQ,CAAC,SAAS;gBACxB,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;gBACzC,eAAe,EAAE,IAAI,CAAC,eAAe;gBACrC,KAAK,EAAE,IAAI,CAAC,IAAI;aACnB;SACJ,CAAC,CAAC;QACH,OAAO,OAAO,CAAC;IACnB,CAAC,CAAC,CAAC,CAAC;IAEJ,uBAAuB;IACvB,IAAI,QAAQ,CAAC,KAAK,IAAI,QAAQ,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9C,cAAG,CAAC,IAAI,CAAC,yCAAyC,GAAG,QAAQ,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC;QAC1F,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;YAClD,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACxC,CAAC,CAAC,CAAC,CAAC;IACR,CAAC;IAED,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE;QAClC,KAAK,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9B,UAAU,EAAE,QAAQ,CAAC,SAAS;KACjC,CAAC,CAAC;IAEH,cAAG,CAAC,IAAI,CAAC,UAAU,QAAQ,qBAAqB,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC;IAE7E,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,CAAA;AAEvE,CAAC"}
1
+ {"version":3,"file":"chunkDocument.js","sourceRoot":"","sources":["../../../src/activities/chunkDocument.ts"],"names":[],"mappings":";;AA+CA,sCAgGC;AA/ID,mDAA2C;AAE3C,wEAAgE;AAEhE,mEAAqG;AAErG,MAAM,kBAAkB,GAAG,mBAAmB,CAAA;AAyCvC,KAAK,UAAU,aAAa,CAAC,OAAyD;IACzF,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAA,kCAAa,EAAsB,OAAO,CAAC,CAAC;IAEvF,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC;IACzB,MAAM,eAAe,GAAG,MAAM,CAAC,eAAe,IAAI,kBAAkB,CAAC;IAErE,cAAG,CAAC,IAAI,CAAC,UAAU,QAAQ,mBAAmB,CAAC,CAAC;IAEhD,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAElE,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAEvF,IAAI,CAAC,IAAI,EAAE,YAAY,EAAE,CAAC;QACtB,cAAG,CAAC,IAAI,CAAC,uCAAuC,GAAG,QAAQ,CAAC,CAAC;QAC7D,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,oBAAoB,EAAE,CAAA;IAC7E,CAAC;IAED,0BAA0B;IAC1B,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QACjB,cAAG,CAAC,IAAI,CAAC,+BAA+B,GAAG,QAAQ,CAAC,CAAC;QACrD,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,eAAe,EAAE,CAAA;IACvE,CAAC;IAED,IAAI,CAAC,KAAK,IAAI,QAAQ,CAAC,KAAK,IAAI,QAAQ,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,QAAQ,CAAC,UAAU,KAAK,QAAQ,CAAC,SAAS,EAAE,CAAC;QACtG,cAAG,CAAC,IAAI,CAAC,0CAA0C,GAAG,QAAQ,CAAC,CAAC;QAChE,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,4CAA4C,EAAE,CAAA;IACrG,CAAC;IAED,uCAAuC;IACvC,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IACvC,MAAM,YAAY,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAEnE,MAAM,GAAG,GAAG,MAAM,IAAA,sDAA8B,EAAC,MAAM,EAAE,eAAe,EAAE,MAAM,EAAE;QAC9E,QAAQ,EAAE,QAAQ;QAClB,OAAO,EAAE,YAAY;KACxB,CAAC,CAAC;IAEH,MAAM,KAAK,GAAG,GAAG,CAAC,MAAM,CAAC,KAAkB,CAAC;IAC5C,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/B,cAAG,CAAC,IAAI,CAAC,gCAAgC,GAAG,QAAQ,EAAE,GAAG,CAAC,CAAC;QAC3D,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,EAAE,EAAE,OAAO,EAAE,gBAAgB,EAAE,CAAA;IACnF,CAAC;IAGD;;OAEG;IACH,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;QAErB,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE;YAE3D,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,IAAI,CAAC,iBAAiB,IAAI,CAAC,IAAI,IAAI,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAE1G,MAAM,QAAQ,GAAG,GAAG,EAAE;gBAClB,IAAI,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC;gBACjC,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;oBACzB,QAAQ,IAAI,QAAQ,CAAC,IAAI,GAAG,GAAG,GAAG,IAAI,CAAC,IAAI,CAAA;gBAC/C,CAAC;gBACD,QAAQ,IAAI,GAAG,GAAG,QAAQ,CAAC,IAAI,GAAG,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC;gBAClD,OAAO,QAAQ,CAAC;YACpB,CAAC,CAAA;YAED,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC;gBACxC,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,MAAM,EAAE,QAAQ;gBAChB,IAAI,EAAE,IAAI;gBACV,QAAQ,EAAE,QAAQ,EAAE;gBACpB,UAAU,EAAE;oBACR,WAAW,EAAE,CAAC,GAAG,CAAC;oBAClB,IAAI,EAAE,QAAQ,CAAC,SAAS;oBACxB,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;oBACzC,eAAe,EAAE,IAAI,CAAC,eAAe;oBACrC,KAAK,EAAE,IAAI,CAAC,IAAI;iBACnB;aACJ,CAAC,CAAC;YACH,OAAO,OAAO,CAAC;QACnB,CAAC,CAAC,CAAC,CAAC;QAEJ,uBAAuB;QACvB,IAAI,QAAQ,CAAC,KAAK,IAAI,QAAQ,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC9C,cAAG,CAAC,IAAI,CAAC,yCAAyC,GAAG,QAAQ,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC;YAC1F,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;gBAClD,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACxC,CAAC,CAAC,CAAC,CAAC;QACR,CAAC;QAED,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE;YAClC,KAAK,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC9B,UAAU,EAAE,QAAQ,CAAC,SAAS;SACjC,CAAC,CAAC;IACP,CAAC;IAED,cAAG,CAAC,IAAI,CAAC,UAAU,QAAQ,qBAAqB,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC;IAE7E,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,CAAA;AAEvE,CAAC"}
@@ -5,12 +5,12 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
5
5
  Object.defineProperty(exports, "__esModule", { value: true });
6
6
  exports.createPdfDocumentFromSource = createPdfDocumentFromSource;
7
7
  const activity_1 = require("@temporalio/activity");
8
+ const node_1 = require("@vertesia/client/node");
8
9
  const fs_1 = __importDefault(require("fs"));
9
10
  const mutool_js_1 = require("../conversion/mutool.js");
10
11
  const ActivityContext_js_1 = require("../dsl/setup/ActivityContext.js");
11
12
  const errors_js_1 = require("../errors.js");
12
13
  const blobs_js_1 = require("../utils/blobs.js");
13
- const memory_js_1 = require("../utils/memory.js");
14
14
  /**
15
15
  * Create a new PDF by extrracting pages from a source PDF
16
16
  * @returns
@@ -45,7 +45,7 @@ async function createPdfDocumentFromSource(payload) {
45
45
  const newPdf = await (0, mutool_js_1.pdfExtractPages)(tmpFile, pages);
46
46
  activity_1.log.info(`PDF created from pages ${pages.join(', ')} `, { newPdf });
47
47
  const name = `pages-${pages.join('-')}.pdf`;
48
- const sourceToUpload = new memory_js_1.NodeStreamSource(fs_1.default.createReadStream(newPdf), name, "application/pdf");
48
+ const sourceToUpload = new node_1.NodeStreamSource(fs_1.default.createReadStream(newPdf), name, "application/pdf");
49
49
  activity_1.log.info(`Uploading file ${newPdf} `);
50
50
  const upload = await client.objects.upload(sourceToUpload);
51
51
  activity_1.log.info(`File uploaded ${upload.source} `);
@@ -1 +1 @@
1
- {"version":3,"file":"createDocumentFromOther.js","sourceRoot":"","sources":["../../../src/activities/createDocumentFromOther.ts"],"names":[],"mappings":";;;;;AA6BA,kEA8DC;AA3FD,mDAA2C;AAE3C,4CAAoB;AACpB,uDAA0D;AAC1D,wEAAgE;AAChE,4CAA+C;AAC/C,gDAAuD;AACvD,kDAAsD;AAkBtD;;;GAGG;AACI,KAAK,UAAU,2BAA2B,CAAC,OAAuE;IACrH,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,IAAA,kCAAa,EAAoC,OAAO,CAAC,CAAC;IACrG,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAE5D,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC;IAC1C,cAAG,CAAC,IAAI,CAAC,0BAA0B,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAE3E,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/B,cAAG,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC;QAC/B,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC;IACzC,CAAC;IAED,IAAI,CAAC,WAAW,EAAE,CAAC;QACf,cAAG,CAAC,KAAK,CAAC,YAAY,QAAQ,YAAY,CAAC,CAAC;QAC5C,MAAM,IAAI,2BAAe,CAAC,YAAY,QAAQ,YAAY,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC5E,CAAC;IAED,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;QAC/B,cAAG,CAAC,KAAK,CAAC,YAAY,QAAQ,gBAAgB,CAAC,CAAC;QAChD,MAAM,IAAI,2BAAe,CAAC,YAAY,QAAQ,gBAAgB,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC;IAChF,CAAC;IAED,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC,WAAW,CAAC,OAAO,CAAC,IAAI,EAAE,UAAU,CAAC,iBAAiB,CAAC,CAAC,EAAE,CAAC;QAC1F,cAAG,CAAC,KAAK,CAAC,YAAY,QAAQ,kBAAkB,CAAC,CAAC;QAClD,MAAM,IAAI,2BAAe,CAAC,YAAY,QAAQ,4BAA4B,WAAW,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC;IACtH,CAAC;IAED,MAAM,UAAU,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC;IAC/E,IAAI,CAAC,UAAU,EAAE,CAAC;QACd,cAAG,CAAC,KAAK,CAAC,QAAQ,MAAM,CAAC,kBAAkB,YAAY,CAAC,CAAC;QACzD,MAAM,IAAI,2BAAe,CAAC,QAAQ,MAAM,CAAC,kBAAkB,YAAY,CAAC,CAAC;IAC7E,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,IAAA,6BAAkB,EAAC,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACrF,MAAM,MAAM,GAAG,MAAM,IAAA,2BAAe,EAAC,OAAO,EAAE,KAAK,CAAC,CAAC;IACrD,cAAG,CAAC,IAAI,CAAC,0BAA0B,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;IACpE,MAAM,IAAI,GAAG,SAAS,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC;IAE5C,MAAM,cAAc,GAAG,IAAI,4BAAgB,CACvC,YAAE,CAAC,gBAAgB,CAAC,MAAM,CAAC,EAC3B,IAAI,EACJ,iBAAiB,CACpB,CAAA;IAED,cAAG,CAAC,IAAI,CAAC,kBAAkB,MAAM,GAAG,CAAC,CAAC;IACtC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;IAC3D,cAAG,CAAC,IAAI,CAAC,iBAAiB,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;IAE5C,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC;QAC1C,IAAI,EAAE,UAAU,CAAC,EAAE;QACnB,IAAI,EAAE,KAAK,IAAI,UAAU,CAAC,IAAI;QAC9B,MAAM,EAAE,QAAQ;QAChB,OAAO,EAAE;YACL,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,IAAI,EAAE,iBAAiB;SAC1B;KACJ,CAAC,CAAC;IAEH,OAAO,EAAE,WAAW,EAAE,SAAS,CAAC,EAAE,EAAE,YAAY,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC;AAGpE,CAAC"}
1
+ {"version":3,"file":"createDocumentFromOther.js","sourceRoot":"","sources":["../../../src/activities/createDocumentFromOther.ts"],"names":[],"mappings":";;;;;AA6BA,kEA8DC;AA3FD,mDAA2C;AAC3C,gDAAyD;AAEzD,4CAAoB;AACpB,uDAA0D;AAC1D,wEAAgE;AAChE,4CAA+C;AAC/C,gDAAuD;AAkBvD;;;GAGG;AACI,KAAK,UAAU,2BAA2B,CAAC,OAAuE;IACrH,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,IAAA,kCAAa,EAAoC,OAAO,CAAC,CAAC;IACrG,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAE5D,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC;IAC1C,cAAG,CAAC,IAAI,CAAC,0BAA0B,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAE3E,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/B,cAAG,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC;QAC/B,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC;IACzC,CAAC;IAED,IAAI,CAAC,WAAW,EAAE,CAAC;QACf,cAAG,CAAC,KAAK,CAAC,YAAY,QAAQ,YAAY,CAAC,CAAC;QAC5C,MAAM,IAAI,2BAAe,CAAC,YAAY,QAAQ,YAAY,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC5E,CAAC;IAED,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;QAC/B,cAAG,CAAC,KAAK,CAAC,YAAY,QAAQ,gBAAgB,CAAC,CAAC;QAChD,MAAM,IAAI,2BAAe,CAAC,YAAY,QAAQ,gBAAgB,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC;IAChF,CAAC;IAED,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC,WAAW,CAAC,OAAO,CAAC,IAAI,EAAE,UAAU,CAAC,iBAAiB,CAAC,CAAC,EAAE,CAAC;QAC1F,cAAG,CAAC,KAAK,CAAC,YAAY,QAAQ,kBAAkB,CAAC,CAAC;QAClD,MAAM,IAAI,2BAAe,CAAC,YAAY,QAAQ,4BAA4B,WAAW,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC;IACtH,CAAC;IAED,MAAM,UAAU,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC;IAC/E,IAAI,CAAC,UAAU,EAAE,CAAC;QACd,cAAG,CAAC,KAAK,CAAC,QAAQ,MAAM,CAAC,kBAAkB,YAAY,CAAC,CAAC;QACzD,MAAM,IAAI,2BAAe,CAAC,QAAQ,MAAM,CAAC,kBAAkB,YAAY,CAAC,CAAC;IAC7E,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,IAAA,6BAAkB,EAAC,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACrF,MAAM,MAAM,GAAG,MAAM,IAAA,2BAAe,EAAC,OAAO,EAAE,KAAK,CAAC,CAAC;IACrD,cAAG,CAAC,IAAI,CAAC,0BAA0B,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;IACpE,MAAM,IAAI,GAAG,SAAS,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC;IAE5C,MAAM,cAAc,GAAG,IAAI,uBAAgB,CACvC,YAAE,CAAC,gBAAgB,CAAC,MAAM,CAAC,EAC3B,IAAI,EACJ,iBAAiB,CACpB,CAAA;IAED,cAAG,CAAC,IAAI,CAAC,kBAAkB,MAAM,GAAG,CAAC,CAAC;IACtC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;IAC3D,cAAG,CAAC,IAAI,CAAC,iBAAiB,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;IAE5C,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC;QAC1C,IAAI,EAAE,UAAU,CAAC,EAAE;QACnB,IAAI,EAAE,KAAK,IAAI,UAAU,CAAC,IAAI;QAC9B,MAAM,EAAE,QAAQ;QAChB,OAAO,EAAE;YACL,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,IAAI,EAAE,iBAAiB;SAC1B;KACJ,CAAC,CAAC;IAEH,OAAO,EAAE,WAAW,EAAE,SAAS,CAAC,EAAE,EAAE,YAAY,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC;AAGpE,CAAC"}
@@ -2,10 +2,11 @@
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.executeInteraction = executeInteraction;
4
4
  exports.executeInteractionFromActivity = executeInteractionFromActivity;
5
- const common_1 = require("@vertesia/common");
6
5
  const activity_1 = require("@temporalio/activity");
6
+ const common_1 = require("@vertesia/common");
7
7
  const projections_js_1 = require("../dsl/projections.js");
8
8
  const ActivityContext_js_1 = require("../dsl/setup/ActivityContext.js");
9
+ const errors_js_1 = require("../errors.js");
9
10
  const tokens_js_1 = require("../utils/tokens.js");
10
11
  //Example:
11
12
  //@ts-ignore
@@ -55,7 +56,14 @@ const JSON = {
55
56
  };
56
57
  async function executeInteraction(payload) {
57
58
  const { client, params } = await (0, ActivityContext_js_1.setupActivity)(payload);
58
- const { interactionName, prompt_data } = params;
59
+ const { interactionName, prompt_data, static_prompt_data: wf_prompt_data } = params;
60
+ if (wf_prompt_data) {
61
+ Object.assign(prompt_data, wf_prompt_data);
62
+ }
63
+ if (!interactionName) {
64
+ activity_1.log.error("Missing interactionName", { params });
65
+ throw new errors_js_1.ActivityParamNotFound("interactionName", payload.activity);
66
+ }
59
67
  if (params.truncate) {
60
68
  const truncate = params.truncate;
61
69
  for (const [key, value] of Object.entries(truncate)) {
@@ -109,9 +117,7 @@ async function executeInteractionFromActivity(client, interactionName, params, p
109
117
  previous_error: previousStudioExecutionRun?.error,
110
118
  };
111
119
  const result_schema = params.result_schema;
112
- if (debug) {
113
- activity_1.log.info(`About to execute interaction ${interactionName}`, { config, data, result_schema, tags });
114
- }
120
+ activity_1.log.debug(`About to execute interaction ${interactionName}`, { config, data, result_schema, tags });
115
121
  const res = await client.interactions.executeByName(interactionName, {
116
122
  config,
117
123
  data,
@@ -1 +1 @@
1
- {"version":3,"file":"executeInteraction.js","sourceRoot":"","sources":["../../../src/activities/executeInteraction.ts"],"names":[],"mappings":";;AAmGA,gDAsBC;AAED,wEAsEC;AAhMD,6CAAuK;AACvK,mDAAyD;AACzD,0DAAsD;AACtD,wEAAgE;AAChE,kDAAoE;AAGpE,UAAU;AACV,YAAY;AACZ,MAAM,IAAI,GAAoB;IAC1B,IAAI,EAAE,oBAAoB;IAC1B,MAAM,EAAE,CAAC,cAAc,EAAE,YAAY,EAAE,WAAW,CAAC;IACnD,MAAM,EAAE;QACJ,YAAY,EAAE,UAAU;QACxB,eAAe,EAAE,iBAAiB;QAClC,KAAK,EAAE,2BAA2B;QAClC,WAAW,EAAE,OAAO;QACpB,UAAU,EAAE,GAAG;QACf,WAAW,EAAE,GAAG;QAChB,IAAI,EAAE,CAAC,MAAM,CAAC;QACd,aAAa,EAAE,0BAA0B;QACzC,WAAW,EAAE;YACT,SAAS,EAAE,cAAc;YACzB,QAAQ,EAAE,kBAAkB;SAC/B;KACJ;IACD,KAAK,EAAE;QACH,SAAS,EAAE;YACP,IAAI,EAAE,UAAU;YAChB,KAAK,EAAE;gBACH,EAAE,EAAE,EAAE,GAAG,EAAE,cAAc,EAAE;aAC9B;YACD,MAAM,EAAE,OAAO;SAClB;QACD,QAAQ,EAAE;YACN,IAAI,EAAE,UAAU;YAChB,KAAK,EAAE,CAAC;YACR,KAAK,EAAE;gBACH,EAAE,EAAE,eAAe;aACtB;YACD,MAAM,EAAE,OAAO;YACf,YAAY,EAAE,OAAO;SACxB;QACD,OAAO,EAAE;YACL,IAAI,EAAE,eAAe;YACrB,KAAK,EAAE,CAAC;YACR,KAAK,EAAE;gBACH,EAAE,EAAE,cAAc;aACrB;YACD,MAAM,EAAE,gBAAgB;SAC3B;KACJ;CACJ,CAAA;AA8CM,KAAK,UAAU,kBAAkB,CAAC,OAA8D;IACnG,MAAM,EACF,MAAM,EAAE,MAAM,EACjB,GAAG,MAAM,IAAA,kCAAa,EAA2B,OAAO,CAAC,CAAC;IAE3D,MAAM,EAAE,eAAe,EAAE,WAAW,EAAE,GAAG,MAAM,CAAC;IAEhD,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;QAClB,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;QACjC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;YAClD,WAAW,CAAC,GAAG,CAAC,GAAG,IAAA,4BAAgB,EAAC,WAAW,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,CAAC;QACjE,CAAC;IACL,CAAC;IAED,MAAM,GAAG,GAAG,MAAM,8BAA8B,CAAC,MAAM,EAAE,eAAe,EAAE,MAAM,EAAE,WAAW,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;IAEnH,OAAO,IAAA,8BAAa,EAAC,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE;QACvC,KAAK,EAAE,GAAG,CAAC,EAAE;QACb,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,MAAM,EAAE,GAAG,CAAC,MAAM;KACrB,CAAC,CAAC;AAEP,CAAC;AAEM,KAAK,UAAU,8BAA8B,CAAC,MAAsB,EAAE,eAAuB,EAAE,MAAkC,EAAE,WAAgB,EAAE,KAAe;IACvK,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC;IAC7B,MAAM,IAAI,GAAG,IAAA,uBAAY,GAAE,CAAC;IAC5B,MAAM,KAAK,GAAG,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC;IAC3C,IAAI,IAAI,GAAG,CAAC,UAAU,EAAE,YAAY,KAAK,EAAE,CAAC,CAAC,CAAC,oBAAoB;IAClE,IAAI,QAAQ,EAAE,CAAC;QACX,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IACjC,CAAC;IAED,IAAI,0BAA0B,GAA6B,SAAS,CAAC;IACrE,IAAI,MAAM,CAAC,sBAAsB,EAAE,CAAC;QAChC,iCAAiC;QACjC,IAAI,IAAI,CAAC,OAAO,GAAG,CAAC,EAAE,CAAC;YACnB,cAAG,CAAC,IAAI,CAAC,sCAAsC,EAAE,EAAE,IAAI,EAAE,CAAC,WAAW,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC;YAClF,MAAM,OAAO,GAAqB;gBAC9B,KAAK,EAAE,EAAE,IAAI,EAAE,CAAC,WAAW,GAAG,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,EAAE;gBAC7D,KAAK,EAAE,CAAC;aACX,CAAC;YACF,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE;gBAC/D,cAAG,CAAC,IAAI,CAAC,gBAAgB,EAAE,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC;gBAC7C,OAAO,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,SAAS,CAAC,CAAC,CAAC,SAAS,CAAA;YAChD,CAAC,CAAC,CAAC;YAEH,IAAI,WAAW,EAAE,CAAC;gBACd,cAAG,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC;gBAChD,0BAA0B,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;YAC5E,CAAC;QACL,CAAC;IACL,CAAC;IACD,IAAI,KAAK,IAAI,0BAA0B,EAAE,KAAK,EAAE,CAAC;QAC7C,cAAG,CAAC,IAAI,CAAC,2BAA2B,EAAE,EAAE,KAAK,EAAE,0BAA0B,EAAE,KAAK,EAAE,CAAC,CAAC;IACxF,CAAC;IAED,MAAM,MAAM,GAAsC;QAC9C,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,aAAa,EAAE,MAAM,CAAC,aAAa;KACtC,CAAA;IACD,MAAM,IAAI,GAAG;QACT,GAAG,WAAW;QACd,cAAc,EAAE,0BAA0B,EAAE,KAAK;KACpD,CAAA;IAED,MAAM,aAAa,GAAG,MAAM,CAAC,aAAa,CAAC;IAE3C,IAAI,KAAK,EAAE,CAAC;QACR,cAAG,CAAC,IAAI,CAAC,gCAAgC,eAAe,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IACvG,CAAC;IAED,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,YAAY,CAAC,aAAa,CAAC,eAAe,EAAE;QACjE,MAAM;QACN,IAAI;QACJ,aAAa;QACb,IAAI;QACJ,MAAM,EAAE,KAAK;KAChB,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;QACb,cAAG,CAAC,KAAK,CAAC,+BAA+B,eAAe,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;QACrE,MAAM,IAAI,KAAK,CAAC,gCAAgC,eAAe,KAAK,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;IACvF,CAAC,CAAC,CAAC;IAEH,IAAI,KAAK,EAAE,CAAC;QACR,cAAG,CAAC,IAAI,CAAC,wBAAwB,eAAe,EAAE,EAAE,GAAG,CAAC,CAAC;IAC7D,CAAC;IAED,IAAI,GAAG,CAAC,KAAK,IAAI,GAAG,CAAC,MAAM,KAAK,2BAAkB,CAAC,MAAM,EAAE,CAAC;QACxD,cAAG,CAAC,KAAK,CAAC,+BAA+B,eAAe,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;QAClF,MAAM,IAAI,KAAK,CAAC,gCAAgC,eAAe,KAAK,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;IACrF,CAAC;IAED,OAAO,GAAG,CAAC;AACf,CAAC"}
1
+ {"version":3,"file":"executeInteraction.js","sourceRoot":"","sources":["../../../src/activities/executeInteraction.ts"],"names":[],"mappings":";;AA+GA,gDA8BC;AAED,wEAoEC;AAlND,mDAAyD;AAEzD,6CAAuK;AACvK,0DAAsD;AACtD,wEAAgE;AAChE,4CAAqD;AACrD,kDAAoE;AAEpE,UAAU;AACV,YAAY;AACZ,MAAM,IAAI,GAAoB;IAC1B,IAAI,EAAE,oBAAoB;IAC1B,MAAM,EAAE,CAAC,cAAc,EAAE,YAAY,EAAE,WAAW,CAAC;IACnD,MAAM,EAAE;QACJ,YAAY,EAAE,UAAU;QACxB,eAAe,EAAE,iBAAiB;QAClC,KAAK,EAAE,2BAA2B;QAClC,WAAW,EAAE,OAAO;QACpB,UAAU,EAAE,GAAG;QACf,WAAW,EAAE,GAAG;QAChB,IAAI,EAAE,CAAC,MAAM,CAAC;QACd,aAAa,EAAE,0BAA0B;QACzC,WAAW,EAAE;YACT,SAAS,EAAE,cAAc;YACzB,QAAQ,EAAE,kBAAkB;SAC/B;KACJ;IACD,KAAK,EAAE;QACH,SAAS,EAAE;YACP,IAAI,EAAE,UAAU;YAChB,KAAK,EAAE;gBACH,EAAE,EAAE,EAAE,GAAG,EAAE,cAAc,EAAE;aAC9B;YACD,MAAM,EAAE,OAAO;SAClB;QACD,QAAQ,EAAE;YACN,IAAI,EAAE,UAAU;YAChB,KAAK,EAAE,CAAC;YACR,KAAK,EAAE;gBACH,EAAE,EAAE,eAAe;aACtB;YACD,MAAM,EAAE,OAAO;YACf,YAAY,EAAE,OAAO;SACxB;QACD,OAAO,EAAE;YACL,IAAI,EAAE,eAAe;YACrB,KAAK,EAAE,CAAC;YACR,KAAK,EAAE;gBACH,EAAE,EAAE,cAAc;aACrB;YACD,MAAM,EAAE,gBAAgB;SAC3B;KACJ;CACJ,CAAA;AAyDM,KAAK,UAAU,kBAAkB,CAAC,OAA8D;IACnG,MAAM,EACF,MAAM,EAAE,MAAM,EACjB,GAAG,MAAM,IAAA,kCAAa,EAA2B,OAAO,CAAC,CAAC;IAE3D,MAAM,EAAE,eAAe,EAAE,WAAW,EAAE,kBAAkB,EAAE,cAAc,EAAE,GAAG,MAAM,CAAC;IACpF,IAAI,cAAc,EAAE,CAAC;QACjB,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;IAC/C,CAAC;IAED,IAAI,CAAC,eAAe,EAAE,CAAC;QACnB,cAAG,CAAC,KAAK,CAAC,yBAAyB,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;QACjD,MAAM,IAAI,iCAAqB,CAAC,iBAAiB,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IACzE,CAAC;IAED,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;QAClB,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;QACjC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;YAClD,WAAW,CAAC,GAAG,CAAC,GAAG,IAAA,4BAAgB,EAAC,WAAW,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,CAAC;QACjE,CAAC;IACL,CAAC;IAED,MAAM,GAAG,GAAG,MAAM,8BAA8B,CAAC,MAAM,EAAE,eAAe,EAAE,MAAM,EAAE,WAAW,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;IAEnH,OAAO,IAAA,8BAAa,EAAC,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE;QACvC,KAAK,EAAE,GAAG,CAAC,EAAE;QACb,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,MAAM,EAAE,GAAG,CAAC,MAAM;KACrB,CAAC,CAAC;AAEP,CAAC;AAEM,KAAK,UAAU,8BAA8B,CAAC,MAAsB,EAAE,eAAuB,EAAE,MAAkC,EAAE,WAAgB,EAAE,KAAe;IACvK,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC;IAC7B,MAAM,IAAI,GAAG,IAAA,uBAAY,GAAE,CAAC;IAC5B,MAAM,KAAK,GAAG,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC;IAC3C,IAAI,IAAI,GAAG,CAAC,UAAU,EAAE,YAAY,KAAK,EAAE,CAAC,CAAC,CAAC,oBAAoB;IAClE,IAAI,QAAQ,EAAE,CAAC;QACX,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IACjC,CAAC;IAED,IAAI,0BAA0B,GAA6B,SAAS,CAAC;IACrE,IAAI,MAAM,CAAC,sBAAsB,EAAE,CAAC;QAChC,iCAAiC;QACjC,IAAI,IAAI,CAAC,OAAO,GAAG,CAAC,EAAE,CAAC;YACnB,cAAG,CAAC,IAAI,CAAC,sCAAsC,EAAE,EAAE,IAAI,EAAE,CAAC,WAAW,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC;YAClF,MAAM,OAAO,GAAqB;gBAC9B,KAAK,EAAE,EAAE,IAAI,EAAE,CAAC,WAAW,GAAG,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,EAAE;gBAC7D,KAAK,EAAE,CAAC;aACX,CAAC;YACF,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE;gBAC/D,cAAG,CAAC,IAAI,CAAC,gBAAgB,EAAE,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC;gBAC7C,OAAO,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,SAAS,CAAC,CAAC,CAAC,SAAS,CAAA;YAChD,CAAC,CAAC,CAAC;YAEH,IAAI,WAAW,EAAE,CAAC;gBACd,cAAG,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC;gBAChD,0BAA0B,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;YAC5E,CAAC;QACL,CAAC;IACL,CAAC;IACD,IAAI,KAAK,IAAI,0BAA0B,EAAE,KAAK,EAAE,CAAC;QAC7C,cAAG,CAAC,IAAI,CAAC,2BAA2B,EAAE,EAAE,KAAK,EAAE,0BAA0B,EAAE,KAAK,EAAE,CAAC,CAAC;IACxF,CAAC;IAED,MAAM,MAAM,GAAsC;QAC9C,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,aAAa,EAAE,MAAM,CAAC,aAAa;KACtC,CAAA;IACD,MAAM,IAAI,GAAG;QACT,GAAG,WAAW;QACd,cAAc,EAAE,0BAA0B,EAAE,KAAK;KACpD,CAAA;IAED,MAAM,aAAa,GAAG,MAAM,CAAC,aAAa,CAAC;IAE3C,cAAG,CAAC,KAAK,CAAC,gCAAgC,eAAe,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IAEpG,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,YAAY,CAAC,aAAa,CAAC,eAAe,EAAE;QACjE,MAAM;QACN,IAAI;QACJ,aAAa;QACb,IAAI;QACJ,MAAM,EAAE,KAAK;KAChB,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;QACb,cAAG,CAAC,KAAK,CAAC,+BAA+B,eAAe,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;QACrE,MAAM,IAAI,KAAK,CAAC,gCAAgC,eAAe,KAAK,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;IACvF,CAAC,CAAC,CAAC;IAEH,IAAI,KAAK,EAAE,CAAC;QACR,cAAG,CAAC,IAAI,CAAC,wBAAwB,eAAe,EAAE,EAAE,GAAG,CAAC,CAAC;IAC7D,CAAC;IAED,IAAI,GAAG,CAAC,KAAK,IAAI,GAAG,CAAC,MAAM,KAAK,2BAAkB,CAAC,MAAM,EAAE,CAAC;QACxD,cAAG,CAAC,KAAK,CAAC,+BAA+B,eAAe,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;QAClF,MAAM,IAAI,KAAK,CAAC,gCAAgC,eAAe,KAAK,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;IACrF,CAAC;IAED,OAAO,GAAG,CAAC;AACf,CAAC"}
@@ -122,14 +122,32 @@ function createResponse(doc, text, status, message) {
122
122
  hasText: !!text,
123
123
  };
124
124
  }
125
- //if file is less than 100KB, check if it looks like text
126
125
  function sniffIfText(buf) {
127
- if (buf.length < 100 * 1024) {
128
- const s = buf.toString('utf8');
129
- if (s.length > 0) {
130
- return true;
126
+ // If file is too large, don't even try
127
+ if (buf.length > 500 * 1024) {
128
+ return false;
129
+ }
130
+ // Count binary/control characters
131
+ let binaryCount = 0;
132
+ const sampleSize = Math.min(buf.length, 1000); // Check first 1000 bytes
133
+ for (let i = 0; i < sampleSize; i++) {
134
+ // Count control characters (except common whitespace)
135
+ const byte = buf[i];
136
+ if ((byte < 32 && ![9, 10, 13].includes(byte)) || byte === 0) {
137
+ binaryCount++;
131
138
  }
132
139
  }
133
- return false;
140
+ // If more than 10% binary/control chars, probably not text
141
+ if (binaryCount / sampleSize > 0.1) {
142
+ return false;
143
+ }
144
+ // Additional check for valid UTF-8 encoding
145
+ try {
146
+ const s = buf.toString('utf8');
147
+ return s.length > 0 && !s.includes('\uFFFD'); // Replacement character
148
+ }
149
+ catch (e) {
150
+ return false;
151
+ }
134
152
  }
135
153
  //# sourceMappingURL=extractDocumentText.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"extractDocumentText.js","sourceRoot":"","sources":["../../../src/activities/extractDocumentText.ts"],"names":[],"mappings":";;AAsBA,kDAyHC;AA/ID,mDAA2C;AAE3C,uDAA0D;AAC1D,uDAAyD;AACzD,wEAAgE;AAChE,4CAA+C;AAC/C,wDAAgF;AAChF,gDAA2D;AAC3D,kDAAiD;AAEjD,YAAY;AACZ,MAAM,IAAI,GAAoB;IAC1B,IAAI,EAAE,qBAAqB;CAC9B,CAAA;AAG6C,CAAC;AAMxC,KAAK,UAAU,mBAAmB,CAAC,OAA+D;IACrG,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAA,kCAAa,EAAC,OAAO,CAAC,CAAC;IAE1D,MAAM,CAAC,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC;QAChC,KAAK,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE;QACxB,KAAK,EAAE,CAAC;QACR,MAAM,EAAE,OAAO;KAClB,CAAC,CAAA;IACF,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,CAAkB,CAAC;IAClC,IAAI,CAAC,GAAG,EAAE,CAAC;QACP,cAAG,CAAC,KAAK,CAAC,YAAY,QAAQ,YAAY,CAAC,CAAC;QAC5C,MAAM,IAAI,2BAAe,CAAC,YAAY,QAAQ,YAAY,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;IACnF,CAAC;IAED,cAAG,CAAC,IAAI,CAAC,8BAA8B,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC;IAGjD,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;QAC7C,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;YACX,OAAO,cAAc,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE,sCAAoB,CAAC,OAAO,EAAE,oCAAoC,CAAC,CAAC;QAC7G,CAAC;aAAM,CAAC;YACJ,OAAO,cAAc,CAAC,GAAG,EAAE,EAAE,EAAE,sCAAoB,CAAC,KAAK,EAAE,yBAAyB,CAAC,CAAC;QAC1F,CAAC;IACL,CAAC;IAED,gDAAgD;IAChD,IAAI,GAAG,CAAC,IAAI,IAAI,GAAG,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,GAAG,CAAC,SAAS,KAAK,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QACxE,OAAO,cAAc,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE,sCAAoB,CAAC,OAAO,EAAE,wBAAwB,CAAC,CAAC;IACjG,CAAC;IAED,IAAI,UAAkB,CAAC;IACvB,IAAI,CAAC;QACD,UAAU,GAAG,MAAM,IAAA,4BAAiB,EAAC,MAAM,EAAE,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACrE,CAAC;IAAC,OAAO,CAAM,EAAE,CAAC;QACd,cAAG,CAAC,KAAK,CAAC,uBAAuB,CAAC,EAAE,CAAC,CAAC;QACtC,OAAO,cAAc,CAAC,GAAG,EAAE,EAAE,EAAE,sCAAoB,CAAC,KAAK,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC;IAC1E,CAAC;IAGD,IAAI,GAAW,CAAC;IAEhB,QAAQ,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAEvB,KAAK,iBAAiB;YAClB,qCAAqC;YACrC,GAAG,GAAG,MAAM,IAAA,2BAAe,EAAC,UAAU,CAAC,CAAC;YACxC,MAAM;QAEV,KAAK,YAAY;YACb,GAAG,GAAG,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAA;YACjC,MAAM;QAEV,MAAM;QACN,KAAK,yEAAyE;YAC1E,GAAG,GAAG,MAAM,IAAA,0BAAc,EAAC,UAAU,EAAE,MAAM,CAAC,CAAC;YAC/C,MAAM;QAEV,MAAM;QACN,KAAK,WAAW;YACZ,GAAG,GAAG,MAAM,IAAA,0BAAc,EAAC,UAAU,EAAE,MAAM,CAAC,CAAC;YAC/C,MAAM;QAEV,cAAc;QACd,KAAK,yCAAyC;YAC1C,GAAG,GAAG,MAAM,IAAA,0BAAc,EAAC,UAAU,EAAE,KAAK,CAAC,CAAC;YAC9C,MAAM;QAEV,KAAK;QACL,KAAK,iBAAiB;YAClB,GAAG,GAAG,MAAM,IAAA,0BAAc,EAAC,UAAU,EAAE,KAAK,CAAC,CAAC;YAC9C,MAAM;QAEV,UAAU;QACV,KAAK,eAAe;YAChB,GAAG,GAAG,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAClC,MAAM;QAEV,KAAK;QACL,KAAK,UAAU;YACX,GAAG,GAAG,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAClC,MAAM;QAEV,YAAY;QACZ,KAAK,wBAAwB;YACzB,GAAG,GAAG,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAClC,MAAM;QAEV,YAAY;QACZ,KAAK,wBAAwB;YACzB,GAAG,GAAG,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAClC,MAAM;QAEV,MAAM;QACN,KAAK,kBAAkB;YACnB,GAAG,GAAG,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAClC,MAAM;QAEV;YACI,IAAI,WAAW,CAAC,UAAU,CAAC,EAAE,CAAC;gBAC1B,GAAG,GAAG,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,6BAA6B;gBAChE,MAAM;YACV,CAAC;YACD,OAAO,cAAc,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,sCAAoB,CAAC,OAAO,EAAE,0BAA0B,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IAC/H,CAAC;IAGD,MAAM,UAAU,GAAG,IAAA,uBAAW,EAAC,GAAG,CAAC,CAAC;IACpC,MAAM,IAAI,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,IAAI,IAAA,cAAG,EAAC,GAAG,CAAC,CAAC;IAE1C,MAAM,UAAU,GAA+B;QAC3C,IAAI,EAAE,GAAG;QACT,SAAS,EAAE,IAAI;QACf,MAAM,EAAE;YACJ,GAAG,UAAU;YACb,IAAI,EAAE,IAAI;SACb;KACJ,CAAA;IAED,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;IAEhD,OAAO,cAAc,CAAC,GAAG,EAAE,GAAG,EAAE,sCAAoB,CAAC,OAAO,CAAC,CAAC;AAClE,CAAC;AAED,SAAS,cAAc,CAAC,GAAkB,EAAE,IAAY,EAAE,MAA4B,EAAE,OAAgB;IACpG,OAAO;QACH,MAAM;QACN,OAAO;QACP,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,GAAG,EAAE,IAAI,CAAC,MAAM;QAChB,QAAQ,EAAE,GAAG,CAAC,EAAE;QAChB,OAAO,EAAE,CAAC,CAAC,IAAI;KAClB,CAAA;AAEL,CAAC;AAGD,yDAAyD;AACzD,SAAS,WAAW,CAAC,GAAW;IAC5B,IAAI,GAAG,CAAC,MAAM,GAAG,GAAG,GAAG,IAAI,EAAE,CAAC;QAC1B,MAAM,CAAC,GAAG,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAC/B,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACf,OAAO,IAAI,CAAC;QAChB,CAAC;IACL,CAAC;IACD,OAAO,KAAK,CAAC;AACjB,CAAC"}
1
+ {"version":3,"file":"extractDocumentText.js","sourceRoot":"","sources":["../../../src/activities/extractDocumentText.ts"],"names":[],"mappings":";;AAsBA,kDAyHC;AA/ID,mDAA2C;AAE3C,uDAA0D;AAC1D,uDAAyD;AACzD,wEAAgE;AAChE,4CAA+C;AAC/C,wDAAgF;AAChF,gDAA2D;AAC3D,kDAAiD;AAEjD,YAAY;AACZ,MAAM,IAAI,GAAoB;IAC1B,IAAI,EAAE,qBAAqB;CAC9B,CAAA;AAG6C,CAAC;AAMxC,KAAK,UAAU,mBAAmB,CAAC,OAA+D;IACrG,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAA,kCAAa,EAAC,OAAO,CAAC,CAAC;IAE1D,MAAM,CAAC,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC;QAChC,KAAK,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE;QACxB,KAAK,EAAE,CAAC;QACR,MAAM,EAAE,OAAO;KAClB,CAAC,CAAA;IACF,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,CAAkB,CAAC;IAClC,IAAI,CAAC,GAAG,EAAE,CAAC;QACP,cAAG,CAAC,KAAK,CAAC,YAAY,QAAQ,YAAY,CAAC,CAAC;QAC5C,MAAM,IAAI,2BAAe,CAAC,YAAY,QAAQ,YAAY,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;IACnF,CAAC;IAED,cAAG,CAAC,IAAI,CAAC,8BAA8B,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC;IAGjD,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;QAC7C,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;YACX,OAAO,cAAc,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE,sCAAoB,CAAC,OAAO,EAAE,oCAAoC,CAAC,CAAC;QAC7G,CAAC;aAAM,CAAC;YACJ,OAAO,cAAc,CAAC,GAAG,EAAE,EAAE,EAAE,sCAAoB,CAAC,KAAK,EAAE,yBAAyB,CAAC,CAAC;QAC1F,CAAC;IACL,CAAC;IAED,gDAAgD;IAChD,IAAI,GAAG,CAAC,IAAI,IAAI,GAAG,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,GAAG,CAAC,SAAS,KAAK,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QACxE,OAAO,cAAc,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE,sCAAoB,CAAC,OAAO,EAAE,wBAAwB,CAAC,CAAC;IACjG,CAAC;IAED,IAAI,UAAkB,CAAC;IACvB,IAAI,CAAC;QACD,UAAU,GAAG,MAAM,IAAA,4BAAiB,EAAC,MAAM,EAAE,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACrE,CAAC;IAAC,OAAO,CAAM,EAAE,CAAC;QACd,cAAG,CAAC,KAAK,CAAC,uBAAuB,CAAC,EAAE,CAAC,CAAC;QACtC,OAAO,cAAc,CAAC,GAAG,EAAE,EAAE,EAAE,sCAAoB,CAAC,KAAK,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC;IAC1E,CAAC;IAGD,IAAI,GAAW,CAAC;IAEhB,QAAQ,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAEvB,KAAK,iBAAiB;YAClB,qCAAqC;YACrC,GAAG,GAAG,MAAM,IAAA,2BAAe,EAAC,UAAU,CAAC,CAAC;YACxC,MAAM;QAEV,KAAK,YAAY;YACb,GAAG,GAAG,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAA;YACjC,MAAM;QAEV,MAAM;QACN,KAAK,yEAAyE;YAC1E,GAAG,GAAG,MAAM,IAAA,0BAAc,EAAC,UAAU,EAAE,MAAM,CAAC,CAAC;YAC/C,MAAM;QAEV,MAAM;QACN,KAAK,WAAW;YACZ,GAAG,GAAG,MAAM,IAAA,0BAAc,EAAC,UAAU,EAAE,MAAM,CAAC,CAAC;YAC/C,MAAM;QAEV,cAAc;QACd,KAAK,yCAAyC;YAC1C,GAAG,GAAG,MAAM,IAAA,0BAAc,EAAC,UAAU,EAAE,KAAK,CAAC,CAAC;YAC9C,MAAM;QAEV,KAAK;QACL,KAAK,iBAAiB;YAClB,GAAG,GAAG,MAAM,IAAA,0BAAc,EAAC,UAAU,EAAE,KAAK,CAAC,CAAC;YAC9C,MAAM;QAEV,UAAU;QACV,KAAK,eAAe;YAChB,GAAG,GAAG,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAClC,MAAM;QAEV,KAAK;QACL,KAAK,UAAU;YACX,GAAG,GAAG,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAClC,MAAM;QAEV,YAAY;QACZ,KAAK,wBAAwB;YACzB,GAAG,GAAG,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAClC,MAAM;QAEV,YAAY;QACZ,KAAK,wBAAwB;YACzB,GAAG,GAAG,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAClC,MAAM;QAEV,MAAM;QACN,KAAK,kBAAkB;YACnB,GAAG,GAAG,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAClC,MAAM;QAEV;YACI,IAAI,WAAW,CAAC,UAAU,CAAC,EAAE,CAAC;gBAC1B,GAAG,GAAG,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,6BAA6B;gBAChE,MAAM;YACV,CAAC;YACD,OAAO,cAAc,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,sCAAoB,CAAC,OAAO,EAAE,0BAA0B,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IAC/H,CAAC;IAGD,MAAM,UAAU,GAAG,IAAA,uBAAW,EAAC,GAAG,CAAC,CAAC;IACpC,MAAM,IAAI,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,IAAI,IAAA,cAAG,EAAC,GAAG,CAAC,CAAC;IAE1C,MAAM,UAAU,GAA+B;QAC3C,IAAI,EAAE,GAAG;QACT,SAAS,EAAE,IAAI;QACf,MAAM,EAAE;YACJ,GAAG,UAAU;YACb,IAAI,EAAE,IAAI;SACb;KACJ,CAAA;IAED,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;IAEhD,OAAO,cAAc,CAAC,GAAG,EAAE,GAAG,EAAE,sCAAoB,CAAC,OAAO,CAAC,CAAC;AAClE,CAAC;AAED,SAAS,cAAc,CAAC,GAAkB,EAAE,IAAY,EAAE,MAA4B,EAAE,OAAgB;IACpG,OAAO;QACH,MAAM;QACN,OAAO;QACP,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,GAAG,EAAE,IAAI,CAAC,MAAM;QAChB,QAAQ,EAAE,GAAG,CAAC,EAAE;QAChB,OAAO,EAAE,CAAC,CAAC,IAAI;KAClB,CAAA;AAEL,CAAC;AAGD,SAAS,WAAW,CAAC,GAAW;IAC5B,uCAAuC;IACvC,IAAI,GAAG,CAAC,MAAM,GAAG,GAAG,GAAG,IAAI,EAAE,CAAC;QAC1B,OAAO,KAAK,CAAC;IACjB,CAAC;IAED,kCAAkC;IAClC,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,yBAAyB;IAExE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,sDAAsD;QACtD,MAAM,IAAI,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;QACpB,IAAI,CAAC,IAAI,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;YAC3D,WAAW,EAAE,CAAC;QAClB,CAAC;IACL,CAAC;IAED,2DAA2D;IAC3D,IAAI,WAAW,GAAG,UAAU,GAAG,GAAG,EAAE,CAAC;QACjC,OAAO,KAAK,CAAC;IACjB,CAAC;IAED,4CAA4C;IAC5C,IAAI,CAAC;QACD,MAAM,CAAC,GAAG,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAC/B,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,wBAAwB;IAC1E,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,OAAO,KAAK,CAAC;IACjB,CAAC;AACL,CAAC"}
@@ -35,24 +35,42 @@ async function generateDocumentProperties(payload) {
35
35
  image: getImageRef() ?? undefined,
36
36
  human_context: project?.configuration?.human_context ?? undefined,
37
37
  };
38
- activity_1.log.info(` Extracting information from object ${objectId} with type ${type.name}`, payload.debug_mode ? { params, } : undefined);
38
+ activity_1.log.info(` Extracting information from object ${objectId} with type ${type.name}`, payload.debug_mode ? { params } : undefined);
39
39
  const infoRes = await (0, executeInteraction_js_1.executeInteractionFromActivity)(client, interactionName, {
40
40
  ...params,
41
41
  include_previous_error: true,
42
42
  result_schema: type.object_schema,
43
43
  }, promptData, payload.debug_mode ?? false);
44
+ const getText = () => {
45
+ if (doc.text) {
46
+ return undefined;
47
+ }
48
+ let text = "";
49
+ if (infoRes.result.title) {
50
+ text += infoRes.result.title + "\n";
51
+ }
52
+ if (infoRes.result.description) {
53
+ text += infoRes.result.description;
54
+ }
55
+ if (text) {
56
+ return text;
57
+ }
58
+ else {
59
+ return undefined;
60
+ }
61
+ };
44
62
  activity_1.log.info(`Extracted information from object ${objectId} with type ${type.name}`, { runId: infoRes.id });
45
63
  await client.objects.update(doc.id, {
46
64
  properties: {
47
65
  ...infoRes.result,
48
- etag: doc.text_etag
66
+ etag: doc.text_etag,
49
67
  },
50
- text: infoRes.result.description ?? undefined,
68
+ text: getText(),
51
69
  generation_run_info: {
52
70
  id: infoRes.id,
53
71
  date: new Date().toISOString(),
54
72
  model: infoRes.modelId,
55
- }
73
+ },
56
74
  });
57
75
  return { status: "completed" };
58
76
  }
@@ -1 +1 @@
1
- {"version":3,"file":"generateDocumentProperties.js","sourceRoot":"","sources":["../../../src/activities/generateDocumentProperties.ts"],"names":[],"mappings":";;AAsBA,gEAsEC;AA3FD,mDAA2C;AAC3C,wEAAgE;AAEhE,mEAAqG;AAErG,MAAM,uBAAuB,GAAG,wBAAwB,CAAA;AAgBjD,KAAK,UAAU,0BAA0B,CAAC,OAAsE;IACnH,MAAM,OAAO,GAAG,MAAM,IAAA,kCAAa,EAAmC,OAAO,CAAC,CAAC;IAC/E,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;IAC7C,MAAM,eAAe,GAAG,MAAM,CAAC,eAAe,IAAI,uBAAuB,CAAC;IAE1E,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,YAAY,EAAE,CAAC;IAE7C,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC7D,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAE7E,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,CAAC,UAAU,IAAI,CAAC,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QAChF,cAAG,CAAC,IAAI,CAAC,UAAU,QAAQ,6BAA6B,CAAC,CAAC;QAC1D,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,CAAA;IACjD,CAAC;IAED,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;QAC/B,cAAG,CAAC,IAAI,CAAC,UAAU,QAAQ,gBAAgB,CAAC,CAAC;QAC7C,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,2BAA2B,EAAE,CAAC;IAC3F,CAAC;IAED,MAAM,WAAW,GAAG,GAAG,EAAE;QACrB,IAAI,GAAG,CAAC,OAAO,EAAE,IAAI,EAAE,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC1C,OAAO,QAAQ,GAAG,GAAG,CAAC,EAAE,CAAC;QAC7B,CAAC;QAED,IAAI,MAAM,CAAC,UAAU,IAAI,GAAG,CAAC,OAAO,EAAE,IAAI,EAAE,UAAU,CAAC,iBAAiB,CAAC,EAAE,CAAC;YACxE,OAAO,QAAQ,GAAG,GAAG,CAAC,EAAE,CAAC;QAC7B,CAAC;QAED,cAAG,CAAC,IAAI,CAAC,UAAU,QAAQ,yBAAyB,CAAC,CAAC;QACtD,OAAO,SAAS,CAAA;IACpB,CAAC,CAAA;IAED,MAAM,UAAU,GAAG;QACf,OAAO,EAAE,GAAG,CAAC,IAAI,IAAI,SAAS;QAC9B,KAAK,EAAE,WAAW,EAAE,IAAI,SAAS;QACjC,aAAa,EAAE,OAAO,EAAE,aAAa,EAAE,aAAa,IAAI,SAAS;KACpE,CAAA;IAED,cAAG,CAAC,IAAI,CAAC,uCAAuC,QAAQ,cAAc,IAAI,CAAC,IAAI,EAAE,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IAEjI,MAAM,OAAO,GAAG,MAAM,IAAA,sDAA8B,EAChD,MAAM,EACN,eAAe,EACf;QACI,GAAG,MAAM;QACT,sBAAsB,EAAE,IAAI;QAC5B,aAAa,EAAE,IAAI,CAAC,aAAa;KACpC,EACD,UAAU,EACV,OAAO,CAAC,UAAU,IAAI,KAAK,CAC9B,CAAC;IAEF,cAAG,CAAC,IAAI,CAAC,qCAAqC,QAAQ,cAAc,IAAI,CAAC,IAAI,EAAE,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC;IACxG,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE;QAChC,UAAU,EAAE;YACR,GAAG,OAAO,CAAC,MAAM;YACjB,IAAI,EAAE,GAAG,CAAC,SAAS;SACtB;QACD,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC,WAAW,IAAI,SAAS;QAC7C,mBAAmB,EAAE;YACjB,EAAE,EAAE,OAAO,CAAC,EAAE;YACd,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YAC9B,KAAK,EAAE,OAAO,CAAC,OAAO;SACzB;KACJ,CAAC,CAAC;IAGH,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC;AAEnC,CAAC"}
1
+ {"version":3,"file":"generateDocumentProperties.js","sourceRoot":"","sources":["../../../src/activities/generateDocumentProperties.ts"],"names":[],"mappings":";;AAsBA,gEA2FC;AAjHD,mDAA2C;AAE3C,wEAAgE;AAEhE,mEAAqG;AAErG,MAAM,uBAAuB,GAAG,wBAAwB,CAAC;AAgBlD,KAAK,UAAU,0BAA0B,CAC5C,OAAsE;IAEtE,MAAM,OAAO,GAAG,MAAM,IAAA,kCAAa,EAAmC,OAAO,CAAC,CAAC;IAC/E,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;IAC7C,MAAM,eAAe,GAAG,MAAM,CAAC,eAAe,IAAI,uBAAuB,CAAC;IAE1E,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,YAAY,EAAE,CAAC;IAE7C,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC7D,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAE7E,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,CAAC,UAAU,IAAI,CAAC,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QAChF,cAAG,CAAC,IAAI,CAAC,UAAU,QAAQ,6BAA6B,CAAC,CAAC;QAC1D,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;IAClD,CAAC;IAED,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;QAC/B,cAAG,CAAC,IAAI,CAAC,UAAU,QAAQ,gBAAgB,CAAC,CAAC;QAC7C,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,2BAA2B,EAAE,CAAC;IAC3F,CAAC;IAED,MAAM,WAAW,GAAG,GAAG,EAAE;QACrB,IAAI,GAAG,CAAC,OAAO,EAAE,IAAI,EAAE,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC1C,OAAO,QAAQ,GAAG,GAAG,CAAC,EAAE,CAAC;QAC7B,CAAC;QAED,IAAI,MAAM,CAAC,UAAU,IAAI,GAAG,CAAC,OAAO,EAAE,IAAI,EAAE,UAAU,CAAC,iBAAiB,CAAC,EAAE,CAAC;YACxE,OAAO,QAAQ,GAAG,GAAG,CAAC,EAAE,CAAC;QAC7B,CAAC;QAED,cAAG,CAAC,IAAI,CAAC,UAAU,QAAQ,yBAAyB,CAAC,CAAC;QACtD,OAAO,SAAS,CAAC;IACrB,CAAC,CAAC;IAEF,MAAM,UAAU,GAAG;QACf,OAAO,EAAE,GAAG,CAAC,IAAI,IAAI,SAAS;QAC9B,KAAK,EAAE,WAAW,EAAE,IAAI,SAAS;QACjC,aAAa,EAAE,OAAO,EAAE,aAAa,EAAE,aAAa,IAAI,SAAS;KACpE,CAAC;IAEF,cAAG,CAAC,IAAI,CACJ,uCAAuC,QAAQ,cAAc,IAAI,CAAC,IAAI,EAAE,EACxE,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,SAAS,CAC9C,CAAC;IAEF,MAAM,OAAO,GAAG,MAAM,IAAA,sDAA8B,EAChD,MAAM,EACN,eAAe,EACf;QACI,GAAG,MAAM;QACT,sBAAsB,EAAE,IAAI;QAC5B,aAAa,EAAE,IAAI,CAAC,aAAa;KACpC,EACD,UAAU,EACV,OAAO,CAAC,UAAU,IAAI,KAAK,CAC9B,CAAC;IAEF,MAAM,OAAO,GAAG,GAAG,EAAE;QACjB,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;YACX,OAAO,SAAS,CAAC;QACrB,CAAC;QACD,IAAI,IAAI,GAAG,EAAE,CAAC;QACd,IAAI,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;YACvB,IAAI,IAAI,OAAO,CAAC,MAAM,CAAC,KAAK,GAAG,IAAI,CAAC;QACxC,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;YAC7B,IAAI,IAAI,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC;QACvC,CAAC;QACD,IAAI,IAAI,EAAE,CAAC;YACP,OAAO,IAAI,CAAC;QAChB,CAAC;aAAM,CAAC;YACJ,OAAO,SAAS,CAAC;QACrB,CAAC;IACL,CAAC,CAAC;IAEF,cAAG,CAAC,IAAI,CAAC,qCAAqC,QAAQ,cAAc,IAAI,CAAC,IAAI,EAAE,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC;IACxG,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE;QAChC,UAAU,EAAE;YACR,GAAG,OAAO,CAAC,MAAM;YACjB,IAAI,EAAE,GAAG,CAAC,SAAS;SACtB;QACD,IAAI,EAAE,OAAO,EAAE;QACf,mBAAmB,EAAE;YACjB,EAAE,EAAE,OAAO,CAAC,EAAE;YACd,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YAC9B,KAAK,EAAE,OAAO,CAAC,OAAO;SACzB;KACJ,CAAC,CAAC;IAEH,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC;AACnC,CAAC"}
@@ -1,45 +1,12 @@
1
1
  "use strict";
2
- var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
- if (k2 === undefined) k2 = k;
4
- var desc = Object.getOwnPropertyDescriptor(m, k);
5
- if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
- desc = { enumerable: true, get: function() { return m[k]; } };
7
- }
8
- Object.defineProperty(o, k2, desc);
9
- }) : (function(o, m, k, k2) {
10
- if (k2 === undefined) k2 = k;
11
- o[k2] = m[k];
12
- }));
13
- var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
- Object.defineProperty(o, "default", { enumerable: true, value: v });
15
- }) : function(o, v) {
16
- o["default"] = v;
17
- });
18
- var __importStar = (this && this.__importStar) || (function () {
19
- var ownKeys = function(o) {
20
- ownKeys = Object.getOwnPropertyNames || function (o) {
21
- var ar = [];
22
- for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
23
- return ar;
24
- };
25
- return ownKeys(o);
26
- };
27
- return function (mod) {
28
- if (mod && mod.__esModule) return mod;
29
- var result = {};
30
- if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
31
- __setModuleDefault(result, mod);
32
- return result;
33
- };
34
- })();
35
2
  Object.defineProperty(exports, "__esModule", { value: true });
36
3
  exports.generateEmbeddings = generateEmbeddings;
37
- const common_1 = require("@vertesia/common");
38
4
  const activity_1 = require("@temporalio/activity");
39
- const tf = __importStar(require("@tensorflow/tfjs-node"));
5
+ const common_1 = require("@vertesia/common");
40
6
  const ActivityContext_js_1 = require("../dsl/setup/ActivityContext.js");
41
7
  const errors_js_1 = require("../errors.js");
42
8
  const blobs_js_1 = require("../utils/blobs.js");
9
+ const chunks_js_1 = require("../utils/chunks.js");
43
10
  const tokens_js_1 = require("../utils/tokens.js");
44
11
  async function generateEmbeddings(payload) {
45
12
  const { params, client, objectId, fetchProject } = await (0, ActivityContext_js_1.setupActivity)(payload);
@@ -101,7 +68,7 @@ async function generateEmbeddings(payload) {
101
68
  }
102
69
  return res;
103
70
  }
104
- async function generateTextEmbeddings({ document, client, type, config }) {
71
+ async function generateTextEmbeddings({ document, client, type, config }, parts) {
105
72
  // if (!force && document.embeddings[type]?.etag === (document.text_etag ?? md5(document.text))) {
106
73
  // return { id: objectId, status: "skipped", message: "embeddings already generated" }
107
74
  // }
@@ -118,6 +85,7 @@ async function generateTextEmbeddings({ document, client, type, config }) {
118
85
  return { id: document.id, status: "failed", message: "no properties found" };
119
86
  }
120
87
  const { environment, model } = config;
88
+ const partDefinitions = parts ?? [];
121
89
  // Count tokens if not already done
122
90
  if (!document.tokens?.count && type === common_1.SupportedEmbeddingTypes.text) {
123
91
  activity_1.log.debug('Updating token count for document: ' + document.id);
@@ -140,65 +108,52 @@ async function generateTextEmbeddings({ document, client, type, config }) {
140
108
  activity_1.log.info(`Generating ${type} embeddings for document ${document.id}`);
141
109
  if (type === common_1.SupportedEmbeddingTypes.text && document.tokens?.count && document.tokens?.count > maxTokens) {
142
110
  activity_1.log.info('Document too large, generating embeddings for parts');
143
- if (!document.parts || document.parts.length === 0) {
144
- return { id: document.id, status: "skipped", message: "no parts found" };
111
+ if (!document.text) {
112
+ return { id: document.id, status: "failed", message: "no text found" };
113
+ }
114
+ if (!partDefinitions || partDefinitions.length === 0) {
115
+ activity_1.log.info('No parts found for document, skipping embeddings generation');
116
+ return { id: document.id, status: "failed", message: "no parts found" };
145
117
  }
146
- const docParts = await Promise.all(document.parts?.map(async (partId) => client.objects.retrieve(partId, "+text +embeddings +properties +tokens")));
118
+ activity_1.log.info('Generating embeddings for parts', { parts: partDefinitions, max_tokens: maxTokens });
119
+ const docParts = (0, chunks_js_1.getContentParts)(document.text, partDefinitions);
147
120
  activity_1.log.info(`Retrieved ${docParts.length} parts`);
148
- const generatePartEmbeddings = async (part, i) => {
121
+ const start = new Date().getTime();
122
+ const generatePartEmbeddings = async (partContent, i) => {
123
+ const localStart = new Date().getTime();
149
124
  try {
150
- activity_1.log.info(`Generating embeddings for part ${part.id}`, { text_len: part.text?.length });
151
- if (!part.text) {
152
- return { id: part.id, number: i, result: null, status: "skipped", message: "no text found" };
125
+ activity_1.log.info(`Generating embeddings for part ${i}`, { text_len: partContent.length });
126
+ if (!partContent) {
127
+ return { id: i, number: i, result: null, status: "skipped", message: "no text found" };
153
128
  }
154
- if (part.tokens?.count && part.tokens.count > maxTokens) {
155
- activity_1.log.info('Part too large, skipping embeddings generation for part', { part: part.id, tokens: part.tokens.count });
156
- return { id: part.id, number: i, result: null, message: "part too large" };
157
- }
158
- const e = await generateEmbeddingsFromStudio(part.text, environment, client, model).catch(e => {
159
- activity_1.log.error('Error generating embeddings for part', { part: part.id, tokens: part.tokens, text_length: part.text?.length, error: e });
129
+ const e = await generateEmbeddingsFromStudio(partContent, environment, client, model).catch(e => {
130
+ activity_1.log.error('Error generating embeddings for part ' + i, { text_length: partContent.length, error: e });
160
131
  return null;
161
132
  });
162
133
  if (!e || !e.values) {
163
- return { id: part.id, number: i, result: null, message: "no embeddings generated" };
134
+ return { id: i, number: i, result: null, message: "no embeddings generated" };
164
135
  }
165
- activity_1.log.info(`Embeddings generated for part ${part.id}, updating object in the store.`);
166
- await client.objects.setEmbedding(part.id, common_1.SupportedEmbeddingTypes.text, {
167
- values: e.values,
168
- model: e.model,
169
- etag: part.text_etag
170
- }).catch(err => {
171
- activity_1.log.info(`Error updating embeddings on part ${part.id}`);
172
- return { id: part.id, number: i, result: null, message: "error setting embeddings on part", error: err.message };
173
- });
174
- activity_1.log.info('Generated embeddings for part: ' + part.id);
175
- return { id: part.id, number: i, result: e };
136
+ if (e.values.length === 0) {
137
+ return { id: i, number: i, result: null, message: "no embeddings generated" };
138
+ }
139
+ activity_1.log.info(`Generated embeddings for part ${i}`, { len: e.values.length, duration: new Date().getTime() - localStart });
140
+ return { inumber: i, result: e };
176
141
  }
177
142
  catch (err) {
178
- activity_1.log.info(`Error generating ${type} embeddings for part ${part.id} of ${document.id}`, { error: err });
179
- return { id: part.id, number: i, result: null, message: "error generating embeddings", error: err.message };
143
+ activity_1.log.info(`Error generating ${type} embeddings for part ${i} of ${document.id}`, { error: err });
144
+ return { number: i, result: null, message: "error generating embeddings", error: err.message };
180
145
  }
181
146
  };
182
- const promises = docParts.map((p, i) => generatePartEmbeddings(p, i));
183
- const res = await Promise.all(promises);
184
- // let i = 0;
185
- // for (const p of docParts) {
186
- // log.info(`Processing part ${p.id}`)
187
- // const r = await generatePartEmbeddings(p, i++);
188
- // res.push(r)
189
- // }
190
- // Filter out parts without embeddings
191
- const validEmbeddings = res.filter(item => item.result !== null);
192
- // Compute the document-level embedding using TensorFlow for attention mechanism
193
- activity_1.log.info('Computing document-level embedding using TF');
194
- const documentEmbedding = computeAttentionEmbedding(validEmbeddings.map(item => item.result.values));
195
- // Save the document-level embedding
147
+ const partEmbeddings = await Promise.all(docParts.map((part, i) => generatePartEmbeddings(part, i)));
148
+ const validPartEmbeddings = partEmbeddings.filter(e => e.result !== null).map(e => e.result);
149
+ const averagedEmbedding = computeAttentionEmbedding(validPartEmbeddings.map(e => e.values));
150
+ activity_1.log.info(`Averaged embeddings for document ${document.id} in ${(new Date().getTime() - start) / 1000} seconds`, { len: averagedEmbedding.length, count: validPartEmbeddings.length, max_tokens: maxTokens });
196
151
  await client.objects.setEmbedding(document.id, type, {
197
- values: documentEmbedding,
198
- model: "attention",
152
+ values: averagedEmbedding,
153
+ model: validPartEmbeddings[0].model,
199
154
  etag: document.text_etag
200
155
  });
201
- return { id: document.id, status: "completed", parts: docParts.map(i => i.id), len: documentEmbedding.length, part_embeddings: res.map(r => { return { id: r.id, status: r.status, error: r.error, message: r.message }; }) };
156
+ activity_1.log.info(`Object ${document.id} embedding set`, { type, len: averagedEmbedding.length });
202
157
  }
203
158
  else {
204
159
  activity_1.log.info(`Generating ${type} embeddings for document`);
@@ -263,30 +218,31 @@ async function generateEmbeddingsFromStudio(text, env, client, model) {
263
218
  throw e;
264
219
  });
265
220
  }
266
- function computeAttentionEmbedding(embeddingsArray, axis = 0) {
267
- if (embeddingsArray.length === 0)
221
+ //Simplified attention mechanism
222
+ // This is a naive implementation and should be replaced with a more sophisticated
223
+ // using tensorflow in a specific package
224
+ function computeAttentionEmbedding(chunkEmbeddings) {
225
+ if (chunkEmbeddings.length === 0)
268
226
  return [];
269
- activity_1.log.info('Computing attention embedding for', { embeddingsArrays: embeddingsArray.map(a => a.length) });
270
227
  const start = new Date().getTime();
271
- // Convert embeddings array to TensorFlow tensor
272
- const embeddingsTensor = tf.tensor(embeddingsArray);
273
- // Initialize trainable attention weights
274
- const attentionWeights = tf.variable(tf.randomNormal([embeddingsArray.length]), true);
275
- // Compute attention scoresje sui
276
- const attentionScores = tf.softmax(attentionWeights);
277
- // Compute weighted sum of embeddings
278
- const weightedEmbeddings = tf.mul(embeddingsTensor.transpose(), attentionScores).transpose();
279
- const documentEmbeddingTensor = tf.sum(weightedEmbeddings, axis);
280
- // Convert the result back to a JavaScript array
281
- const documentEmbedding = documentEmbeddingTensor.arraySync();
282
- const duration = (new Date().getTime() - start);
283
- activity_1.log.info(`Computed attention embeddings in ${duration}ms - array size: ${documentEmbedding.length}`, { length: documentEmbedding.length });
284
- // Clean up tensors
285
- embeddingsTensor.dispose();
286
- attentionWeights.dispose();
287
- attentionScores.dispose();
288
- weightedEmbeddings.dispose();
289
- documentEmbeddingTensor.dispose();
228
+ // Generate random attention weights
229
+ const attentionWeights = chunkEmbeddings.map(() => Math.random());
230
+ // Apply softmax to get attention scores
231
+ const expWeights = attentionWeights.map(w => Math.exp(w));
232
+ const sumExpWeights = expWeights.reduce((sum, val) => sum + val, 0);
233
+ const attentionScores = expWeights.map(w => w / sumExpWeights);
234
+ // Get embedding dimension
235
+ const embeddingDim = chunkEmbeddings[0].length;
236
+ // Initialize document embedding
237
+ const documentEmbedding = new Array(embeddingDim).fill(0);
238
+ // Weighted sum of embeddings
239
+ for (let i = 0; i < chunkEmbeddings.length; i++) {
240
+ for (let j = 0; j < embeddingDim; j++) {
241
+ documentEmbedding[j] += chunkEmbeddings[i][j] * attentionScores[i];
242
+ }
243
+ }
244
+ const duration = new Date().getTime() - start;
245
+ console.log(`Computed document embedding in ${duration}ms for ${chunkEmbeddings.length} chunks`);
290
246
  return documentEmbedding;
291
247
  }
292
248
  //# sourceMappingURL=generateEmbeddings.js.map