@hassangameryt/freeflow 0.5.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (301) hide show
  1. package/.claude-plugin/marketplace.json +1 -1
  2. package/.claude-plugin/plugin.json +2 -2
  3. package/.codex-plugin/plugin.json +4 -5
  4. package/README.md +141 -163
  5. package/capabilities/cognitive-routing/SKILL.md +387 -0
  6. package/capabilities/context-virtualization/SKILL.md +103 -0
  7. package/capabilities/conversation-history/SKILL.md +53 -0
  8. package/command-surface.json +6 -37
  9. package/hooks/freeflow-runtime-context.mjs +50 -541
  10. package/hooks/hooks.json +1 -26
  11. package/hooks/tests/check-runtime-context-hook.sh +114 -308
  12. package/package.json +27 -17
  13. package/pi-extension/dist/cognitive-routing/commands.js +96 -0
  14. package/pi-extension/dist/cognitive-routing/config.js +232 -0
  15. package/pi-extension/dist/cognitive-routing/controller.js +1292 -0
  16. package/pi-extension/dist/cognitive-routing/history.js +451 -0
  17. package/pi-extension/dist/cognitive-routing/host.js +188 -0
  18. package/pi-extension/dist/cognitive-routing/pi-routing-session.js +414 -0
  19. package/pi-extension/dist/cognitive-routing/pi-session-control.js +320 -0
  20. package/pi-extension/dist/cognitive-routing/runtime.js +43 -0
  21. package/pi-extension/dist/cognitive-routing/tool.js +212 -0
  22. package/pi-extension/dist/cognitive-routing/types.js +6 -0
  23. package/pi-extension/dist/context-virtualization/commands.js +89 -0
  24. package/pi-extension/dist/context-virtualization/projector.js +42 -0
  25. package/pi-extension/dist/context-virtualization/runtime.js +356 -0
  26. package/pi-extension/dist/context-virtualization/state.js +57 -0
  27. package/pi-extension/dist/context-virtualization/types.js +10 -0
  28. package/pi-extension/dist/conversation-history/passages.js +66 -0
  29. package/pi-extension/dist/conversation-history/ranking.js +105 -0
  30. package/pi-extension/dist/conversation-history/runtime.js +226 -0
  31. package/pi-extension/dist/conversation-history/search.js +56 -0
  32. package/pi-extension/dist/conversation-history/types.js +1 -0
  33. package/pi-extension/dist/freeflow-context/resolver.js +120 -0
  34. package/pi-extension/dist/freeflow-context/runtime.js +80 -0
  35. package/pi-extension/dist/freeflow-context/source-projector.js +95 -0
  36. package/pi-extension/dist/freeflow-context/tool.js +308 -0
  37. package/pi-extension/dist/freeflow-context/types.js +12 -0
  38. package/pi-extension/dist/freeflow-context/visibility.js +28 -0
  39. package/pi-extension/dist/index.js +564 -129
  40. package/pi-extension/dist/runtime/runtime-context.js +357 -470
  41. package/pi-extension/dist/runtime/runtime-identity.js +5 -0
  42. package/pi-extension/dist/settings/settings-tui.js +78 -0
  43. package/pi-extension/dist/settings/settings-ui.js +572 -826
  44. package/pi-extension/dist/ui/text-component.js +57 -0
  45. package/pi-extension/src/cognitive-routing/commands.ts +129 -0
  46. package/pi-extension/src/cognitive-routing/config.ts +303 -0
  47. package/pi-extension/src/cognitive-routing/controller.ts +1628 -0
  48. package/pi-extension/src/cognitive-routing/history.ts +650 -0
  49. package/pi-extension/src/cognitive-routing/host.ts +266 -0
  50. package/pi-extension/src/cognitive-routing/pi-routing-session.ts +538 -0
  51. package/pi-extension/src/cognitive-routing/pi-session-control.ts +444 -0
  52. package/pi-extension/src/cognitive-routing/runtime.ts +57 -0
  53. package/pi-extension/src/cognitive-routing/tool.ts +273 -0
  54. package/pi-extension/src/cognitive-routing/types.ts +89 -0
  55. package/pi-extension/src/context-virtualization/commands.ts +111 -0
  56. package/pi-extension/src/context-virtualization/projector.ts +62 -0
  57. package/pi-extension/src/context-virtualization/runtime.ts +406 -0
  58. package/pi-extension/src/context-virtualization/state.ts +79 -0
  59. package/pi-extension/src/context-virtualization/types.ts +92 -0
  60. package/pi-extension/src/conversation-history/passages.ts +81 -0
  61. package/pi-extension/src/conversation-history/ranking.ts +155 -0
  62. package/pi-extension/src/conversation-history/runtime.ts +250 -0
  63. package/pi-extension/src/conversation-history/search.ts +79 -0
  64. package/pi-extension/src/conversation-history/types.ts +31 -0
  65. package/pi-extension/src/freeflow-context/resolver.ts +139 -0
  66. package/pi-extension/src/freeflow-context/runtime.ts +100 -0
  67. package/pi-extension/src/freeflow-context/source-projector.ts +106 -0
  68. package/pi-extension/src/freeflow-context/tool.ts +360 -0
  69. package/pi-extension/src/freeflow-context/types.ts +70 -0
  70. package/pi-extension/src/freeflow-context/visibility.ts +53 -0
  71. package/pi-extension/src/index.ts +597 -135
  72. package/pi-extension/src/runtime/runtime-context.ts +373 -491
  73. package/pi-extension/src/runtime/runtime-identity.ts +8 -0
  74. package/pi-extension/src/settings/settings-tui.ts +93 -0
  75. package/pi-extension/src/settings/settings-ui.ts +679 -910
  76. package/pi-extension/src/ui/text-component.ts +66 -0
  77. package/pi-extension/tests/cognitive-routing/commands.test.js +134 -0
  78. package/pi-extension/tests/cognitive-routing/controller.test.js +1128 -0
  79. package/pi-extension/tests/cognitive-routing/extension-lifecycle.test.js +1142 -0
  80. package/pi-extension/tests/cognitive-routing/history.test.js +274 -0
  81. package/pi-extension/tests/cognitive-routing/host-fixture.js +4 -0
  82. package/pi-extension/tests/cognitive-routing/host-identity.test.js +63 -0
  83. package/pi-extension/tests/cognitive-routing/pi-normal-lifecycle.test.js +434 -0
  84. package/pi-extension/tests/cognitive-routing/pi-session-control.test.js +174 -0
  85. package/pi-extension/tests/cognitive-routing/pi-session-history.test.js +91 -0
  86. package/pi-extension/tests/cognitive-routing/profile-resolution.test.js +279 -0
  87. package/pi-extension/tests/cognitive-routing/tool.test.js +230 -0
  88. package/pi-extension/tests/context-virtualization/extension.test.js +749 -0
  89. package/pi-extension/tests/context-virtualization/projector.test.js +110 -0
  90. package/pi-extension/tests/context-virtualization/runtime.test.js +216 -0
  91. package/pi-extension/tests/conversation-history/runtime.test.js +98 -0
  92. package/pi-extension/tests/conversation-history/search.test.js +97 -0
  93. package/pi-extension/tests/integration/mode-free-core.test.js +218 -0
  94. package/pi-extension/tests/integration/pi-extension.test.js +532 -3701
  95. package/pi-extension/tests/integration/prompt-architecture.test.js +350 -0
  96. package/pi-extension/tests/integration/unaffected-regressions.test.js +411 -0
  97. package/runtime/prompts/cognitive-routing.md +10 -0
  98. package/runtime/prompts/context-virtualization.md +3 -0
  99. package/runtime/prompts/conversation-history.md +3 -0
  100. package/runtime/prompts/core.md +84 -0
  101. package/runtime/prompts/interaction-contract.md +11 -0
  102. package/skills/action-selection/SKILL.md +140 -0
  103. package/skills/action-selection/references/trajectory-stalls.md +71 -0
  104. package/skills/bypass/SKILL.md +109 -38
  105. package/skills/commit-work/SKILL.md +94 -55
  106. package/skills/commit-work/references/git-state-edges.md +94 -0
  107. package/skills/commit-work/references/simple-push.md +92 -0
  108. package/skills/commit-work/references/staging-decisions.md +79 -46
  109. package/skills/decision-gate/SKILL.md +80 -22
  110. package/skills/design-for-depth/SKILL.md +206 -60
  111. package/skills/design-for-depth/references/design-pressure-signals.md +108 -240
  112. package/skills/design-for-depth/references/interface-design-loop.md +77 -97
  113. package/skills/design-for-depth/references/module-and-dependency-design.md +147 -0
  114. package/skills/design-for-depth/references/software-design-philosophy.md +128 -174
  115. package/skills/design-for-depth/references/state-and-failure-boundaries.md +136 -0
  116. package/skills/diagnose-failure/SKILL.md +200 -48
  117. package/skills/diagnose-failure/references/diagnostic-loop-catalog.md +18 -6
  118. package/skills/diagnose-failure/references/{flaky-and-performance.md → flaky-and-performance-diagnosis.md} +13 -5
  119. package/skills/discuss/SKILL.md +119 -65
  120. package/skills/discuss/references/discussion-continuity.md +123 -0
  121. package/skills/evaluate-skill/SKILL.md +166 -52
  122. package/skills/evaluate-skill/references/{definition-schema.md → evaluation-definition-schema.md} +43 -3
  123. package/skills/evaluate-skill/references/evaluation-design.md +56 -30
  124. package/skills/evaluate-skill/references/execution-and-evidence.md +8 -3
  125. package/skills/evaluate-skill/references/review-and-revision.md +62 -21
  126. package/skills/evaluate-skill/scripts/lib/definitions.mjs +114 -3
  127. package/skills/evaluate-skill/scripts/lib/grade-interaction.mjs +156 -1
  128. package/skills/evaluate-skill/scripts/lib/grade.mjs +9 -1
  129. package/skills/evaluate-skill/scripts/lib/pi.mjs +147 -32
  130. package/skills/evaluate-skill/scripts/lib/rpc.mjs +4 -4
  131. package/skills/evaluate-skill/scripts/lib/runner.mjs +36 -14
  132. package/skills/evaluate-skill/scripts/lib/sandbox.mjs +92 -0
  133. package/skills/evaluate-skill/scripts/pi-guard.mjs +1 -1
  134. package/skills/evaluate-skill/scripts/pi-observer.mjs +66 -0
  135. package/skills/evaluate-skill/scripts/skill-eval.mjs +6 -5
  136. package/skills/execute-work/SKILL.md +99 -57
  137. package/skills/execute-work/references/code-practices.md +4 -4
  138. package/skills/execute-work/references/execute-work-edges.md +62 -0
  139. package/skills/finish-branch/SKILL.md +2 -2
  140. package/skills/finish-branch/references/integration-options.md +1 -1
  141. package/skills/handoff/SKILL.md +3 -3
  142. package/skills/launch-work/SKILL.md +1 -1
  143. package/skills/launch-work/references/launch-readiness.md +2 -2
  144. package/skills/migration-work/SKILL.md +2 -2
  145. package/skills/migration-work/references/migration-lifecycle.md +1 -1
  146. package/skills/release-work/SKILL.md +1 -1
  147. package/skills/release-work/references/release-evidence.md +1 -1
  148. package/skills/review-artifact/SKILL.md +129 -66
  149. package/skills/review-artifact/references/adjudicate-artifact-review.md +101 -0
  150. package/skills/review-artifact/references/independent-artifact-reviewer-contract.md +159 -0
  151. package/skills/review-work/SKILL.md +113 -68
  152. package/skills/review-work/references/adjudicate-work-review.md +97 -0
  153. package/skills/review-work/references/independent-work-reviewer-contract.md +139 -0
  154. package/skills/review-work/references/security-risk-lens.md +18 -16
  155. package/skills/setup-freeflow/SKILL.md +16 -17
  156. package/skills/setup-freeflow/references/activation-contract.md +24 -30
  157. package/skills/setup-freeflow/references/host-setup.md +6 -7
  158. package/skills/simplify-code/SKILL.md +97 -39
  159. package/skills/simplify-code/references/simplification-patterns.md +1 -1
  160. package/skills/tdd/SKILL.md +100 -53
  161. package/skills/tdd/references/test-design.md +3 -3
  162. package/skills/track-work/SKILL.md +195 -58
  163. package/skills/track-work/references/working-record-format.md +515 -0
  164. package/skills/track-work/scripts/lib/document.mjs +713 -0
  165. package/skills/track-work/scripts/lib/format.mjs +166 -0
  166. package/skills/track-work/scripts/lib/store.mjs +233 -0
  167. package/skills/track-work/scripts/lib/transitions.mjs +790 -0
  168. package/skills/track-work/scripts/lib/views.mjs +40 -0
  169. package/skills/track-work/scripts/working-record.mjs +316 -0
  170. package/skills/verify-work/SKILL.md +57 -19
  171. package/skills/verify-work/references/browser-runtime-evidence.md +2 -2
  172. package/skills/verify-work/references/integration-evidence.md +1 -7
  173. package/skills/verify-work/references/performance-evidence.md +2 -2
  174. package/skills/workflow/SKILL.md +137 -55
  175. package/skills/workflow/references/domain-skill-composition.md +26 -40
  176. package/skills/write-plan/SKILL.md +1 -1
  177. package/skills/write-skill/SKILL.md +218 -61
  178. package/skills/write-skill/references/activation-boundaries.md +63 -21
  179. package/skills/write-skill/references/agent-first-instructions.md +61 -27
  180. package/skills/write-skill/references/progressive-disclosure.md +97 -33
  181. package/skills/write-skill/references/skill-dependency-graphs.md +190 -0
  182. package/capabilities/output-router/SKILL.md +0 -206
  183. package/capabilities/output-router/references/safety-policy.md +0 -40
  184. package/capabilities/output-router/references/setup.md +0 -110
  185. package/pi-extension/dist/output-router/host-producer-identification.js +0 -232
  186. package/pi-extension/dist/output-router/native-safety-net.js +0 -216
  187. package/pi-extension/dist/output-router/observed-tool-routing.js +0 -122
  188. package/pi-extension/dist/output-router/renderers.js +0 -1023
  189. package/pi-extension/dist/output-router/router-tools.js +0 -554
  190. package/pi-extension/dist/output-router/schemas.js +0 -383
  191. package/pi-extension/dist/output-router/status.js +0 -744
  192. package/pi-extension/dist/output-router/utils.js +0 -502
  193. package/pi-extension/src/output-router/host-producer-identification.ts +0 -256
  194. package/pi-extension/src/output-router/native-safety-net.ts +0 -248
  195. package/pi-extension/src/output-router/observed-tool-routing.ts +0 -134
  196. package/pi-extension/src/output-router/renderers.ts +0 -1108
  197. package/pi-extension/src/output-router/router-tools.ts +0 -579
  198. package/pi-extension/src/output-router/schemas.ts +0 -387
  199. package/pi-extension/src/output-router/status.ts +0 -791
  200. package/pi-extension/src/output-router/utils.ts +0 -554
  201. package/pi-extension/tests/output-router/observed-producers.test.js +0 -132
  202. package/pi-extension/tests/output-router/search.test.js +0 -722
  203. package/router/dist/benchmarks/benchmark-harness.d.ts +0 -35
  204. package/router/dist/benchmarks/benchmark-harness.js +0 -92
  205. package/router/dist/benchmarks/benchmarks.d.ts +0 -109
  206. package/router/dist/benchmarks/benchmarks.js +0 -979
  207. package/router/dist/benchmarks/codex-qa-benchmarks.d.ts +0 -95
  208. package/router/dist/benchmarks/codex-qa-benchmarks.js +0 -500
  209. package/router/dist/benchmarks/command-benchmarks.d.ts +0 -143
  210. package/router/dist/benchmarks/command-benchmarks.js +0 -698
  211. package/router/dist/benchmarks/context-mode-normalized-benchmarks.d.ts +0 -82
  212. package/router/dist/benchmarks/context-mode-normalized-benchmarks.js +0 -773
  213. package/router/dist/benchmarks/context-mode-real-deep-benchmark.d.ts +0 -132
  214. package/router/dist/benchmarks/context-mode-real-deep-benchmark.js +0 -1380
  215. package/router/dist/benchmarks/context-saving-baseline.d.ts +0 -60
  216. package/router/dist/benchmarks/context-saving-baseline.js +0 -470
  217. package/router/dist/benchmarks/index-benchmarks.d.ts +0 -103
  218. package/router/dist/benchmarks/index-benchmarks.js +0 -862
  219. package/router/dist/benchmarks/storage-policy-benchmarks.d.ts +0 -101
  220. package/router/dist/benchmarks/storage-policy-benchmarks.js +0 -618
  221. package/router/dist/benchmarks/vault-index-benchmarks.d.ts +0 -56
  222. package/router/dist/benchmarks/vault-index-benchmarks.js +0 -322
  223. package/router/dist/config/config.d.ts +0 -103
  224. package/router/dist/config/config.js +0 -632
  225. package/router/dist/config/router-contract.d.ts +0 -10
  226. package/router/dist/config/router-contract.js +0 -54
  227. package/router/dist/config/schema.d.ts +0 -27
  228. package/router/dist/config/schema.js +0 -648
  229. package/router/dist/config/types.d.ts +0 -573
  230. package/router/dist/config/types.js +0 -41
  231. package/router/dist/evidence/bounded-evidence.d.ts +0 -40
  232. package/router/dist/evidence/bounded-evidence.js +0 -323
  233. package/router/dist/evidence/evidence-range-selector.d.ts +0 -24
  234. package/router/dist/evidence/evidence-range-selector.js +0 -232
  235. package/router/dist/evidence/evidence-search.d.ts +0 -27
  236. package/router/dist/evidence/evidence-search.js +0 -490
  237. package/router/dist/evidence/evidence.d.ts +0 -38
  238. package/router/dist/evidence/evidence.js +0 -200
  239. package/router/dist/evidence/failure-contracts.d.ts +0 -37
  240. package/router/dist/evidence/failure-contracts.js +0 -166
  241. package/router/dist/evidence/line-ranges.d.ts +0 -24
  242. package/router/dist/evidence/line-ranges.js +0 -22
  243. package/router/dist/experiments/local-index.d.ts +0 -64
  244. package/router/dist/experiments/local-index.js +0 -581
  245. package/router/dist/index.d.ts +0 -30
  246. package/router/dist/index.js +0 -32
  247. package/router/dist/local/local-traversal.d.ts +0 -18
  248. package/router/dist/local/local-traversal.js +0 -334
  249. package/router/dist/processing/engine.d.ts +0 -132
  250. package/router/dist/processing/engine.js +0 -533
  251. package/router/dist/processing/reducers.d.ts +0 -280
  252. package/router/dist/processing/reducers.js +0 -1673
  253. package/router/dist/processing/renderers.d.ts +0 -31
  254. package/router/dist/processing/renderers.js +0 -195
  255. package/router/dist/processing/scripts.d.ts +0 -91
  256. package/router/dist/processing/scripts.js +0 -386
  257. package/router/dist/repo/repo-traversal.d.ts +0 -18
  258. package/router/dist/repo/repo-traversal.js +0 -245
  259. package/router/dist/routing/observed-reducers.d.ts +0 -41
  260. package/router/dist/routing/observed-reducers.js +0 -661
  261. package/router/dist/routing/observed-routing.d.ts +0 -26
  262. package/router/dist/routing/observed-routing.js +0 -378
  263. package/router/dist/routing/parsers.d.ts +0 -16
  264. package/router/dist/routing/parsers.js +0 -521
  265. package/router/dist/routing/run-filters.d.ts +0 -60
  266. package/router/dist/routing/run-filters.js +0 -238
  267. package/router/dist/routing/run-reducers.d.ts +0 -49
  268. package/router/dist/routing/run-reducers.js +0 -122
  269. package/router/dist/sandbox/adapter-roots.d.ts +0 -46
  270. package/router/dist/sandbox/adapter-roots.js +0 -66
  271. package/router/dist/sandbox/eryx-python-adapter.d.ts +0 -13
  272. package/router/dist/sandbox/eryx-python-adapter.js +0 -868
  273. package/router/dist/sandbox/jq-wasm-adapter.d.ts +0 -9
  274. package/router/dist/sandbox/jq-wasm-adapter.js +0 -429
  275. package/router/dist/sandbox/quickjs-wasi-adapter.d.ts +0 -13
  276. package/router/dist/sandbox/quickjs-wasi-adapter.js +0 -384
  277. package/router/dist/sandbox/script-sandbox.d.ts +0 -134
  278. package/router/dist/sandbox/script-sandbox.js +0 -337
  279. package/router/dist/setup/script-transform-adapters.d.ts +0 -32
  280. package/router/dist/setup/script-transform-adapters.js +0 -278
  281. package/router/dist/tools/batch.d.ts +0 -33
  282. package/router/dist/tools/batch.js +0 -586
  283. package/router/dist/tools/run.d.ts +0 -62
  284. package/router/dist/tools/run.js +0 -1304
  285. package/router/dist/tools/search.d.ts +0 -66
  286. package/router/dist/tools/search.js +0 -1364
  287. package/router/dist/transform/engine.d.ts +0 -168
  288. package/router/dist/transform/engine.js +0 -2072
  289. package/router/dist/vault/vault-index.d.ts +0 -125
  290. package/router/dist/vault/vault-index.js +0 -416
  291. package/router/dist/vault/vault.d.ts +0 -138
  292. package/router/dist/vault/vault.js +0 -630
  293. package/runtime/interaction-contract.md +0 -16
  294. package/skills/discuss/references/checkpoints.md +0 -66
  295. package/skills/execute-work/references/execution-loop.md +0 -77
  296. package/skills/mode-contract/SKILL.md +0 -104
  297. package/skills/review-artifact/references/reviewer-prompt.md +0 -148
  298. package/skills/review-work/references/reviewer-prompt.md +0 -134
  299. package/skills/track-work/references/working-record-schema.md +0 -378
  300. package/skills/workflow/references/workflow-loop.md +0 -92
  301. /package/skills/handoff/references/{templates.md → handoff-templates.md} +0 -0
@@ -1,2072 +0,0 @@
1
- import { createHash } from "node:crypto";
2
- import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
3
- import { tmpdir } from "node:os";
4
- import { join } from "node:path";
5
- import {
6
- DEFAULT_ROUTER_THRESHOLDS,
7
- DEFAULT_SCRIPT_TRANSFORM_CONFIG,
8
- MAX_SCRIPT_TRANSFORM_LIMITS,
9
- SCRIPT_TRANSFORM_LANGUAGES,
10
- } from "../config/config.js";
11
- import { assembleTextEvidence, byteLength, countLines, splitLines } from "../evidence/evidence.js";
12
- import { selectScriptSandboxAdapter } from "../sandbox/script-sandbox.js";
13
- import {
14
- transformAdapterUnavailableFailure,
15
- transformSourceUnavailableFailure,
16
- transformValidationFailure,
17
- transformExecutionFailure,
18
- scriptTransformDisabledFailure,
19
- storageFailure,
20
- } from "../evidence/failure-contracts.js";
21
- import { createVault, readOutputText, readVaultRecord, storeTextOutput } from "../vault/vault.js";
22
- export const TRANSFORM_ENGINE_IMPLEMENTATION = "shared-transform-engine-v1";
23
- const PRESERVE_MODES = new Set(["summary", "important", "full"]);
24
- const OPERATION_KINDS = new Set([
25
- "regexFilter",
26
- "countMatches",
27
- "jsonExtract",
28
- "groupByRegex",
29
- "dedupe",
30
- "topN",
31
- "extractUrls",
32
- "extractCitations",
33
- "lineStats",
34
- "sizeStats",
35
- "script",
36
- ]);
37
- const REGEX_FLAGS = new Set(["g", "i", "m", "s", "u"]);
38
- const DEFAULT_CONTEXT_LINES = 0;
39
- const DEFAULT_MAX_MATCHES = 50;
40
- const MAX_CONTEXT_LINES = 20;
41
- const MAX_MATCHES = 1_000;
42
- const DEFAULT_MAX_GROUPS = 100;
43
- const DEFAULT_MAX_LINES_PER_GROUP = 20;
44
- const DEFAULT_MAX_DEDUPE_LINES = 1_000;
45
- const MAX_GROUPS = 1_000;
46
- const MAX_LINES_PER_GROUP = 1_000;
47
- const MAX_DEDUPE_LINES = 10_000;
48
- const MAX_TOP_N_LIMIT = 1_000;
49
- const DEFAULT_MAX_EXTRACT_MATCHES = 1_000;
50
- const MAX_EXTRACT_MATCHES = 10_000;
51
- const URL_PATTERN = /https?:\/\/[^\s<>"'\])}]+/gi;
52
- const SCRIPT_ALIAS_PATTERN = /^[A-Za-z][A-Za-z0-9_-]{0,63}$/;
53
- const MAX_SCRIPT_CODE_BYTES = 256 * 1024;
54
- export function validateTransformInput(value) {
55
- const issues = [];
56
- if (!isRecord(value)) {
57
- return { ok: false, issues: [{ path: "$", message: "Expected transform input object." }] };
58
- }
59
- const operationKind = isRecord(value.operation) ? value.operation.kind : undefined;
60
- validateTransformOperation(value.operation, "$.operation", issues);
61
- if (operationKind === "script") {
62
- validateScriptTransformSources(value.sources, "$.sources", issues);
63
- validateScriptTransformLimits(value.limits, "$.limits", issues);
64
- } else {
65
- validateTransformSource(value.source, "$.source", issues);
66
- }
67
- if (value.preserve !== undefined && (typeof value.preserve !== "string" || !PRESERVE_MODES.has(value.preserve))) {
68
- issues.push({ path: "$.preserve", message: "Expected preserve mode summary, important, or full." });
69
- }
70
- if (issues.length > 0) {
71
- return { ok: false, issues };
72
- }
73
- if (operationKind === "script") {
74
- const input = {
75
- sources: value.sources,
76
- operation: value.operation,
77
- };
78
- if (value.limits !== undefined) {
79
- input.limits = value.limits;
80
- }
81
- if (value.preserve !== undefined) {
82
- input.preserve = value.preserve;
83
- }
84
- return { ok: true, value: input };
85
- }
86
- const input = {
87
- source: value.source,
88
- operation: value.operation,
89
- };
90
- if (value.preserve !== undefined) {
91
- input.preserve = value.preserve;
92
- }
93
- return { ok: true, value: input };
94
- }
95
- export async function freeflowTransform(options) {
96
- return executeTransform(options);
97
- }
98
- async function executeTransform(options) {
99
- const preserve = options.preserve ?? "important";
100
- const inputValidation = validateTransformInput(options);
101
- if (!inputValidation.ok) {
102
- return transformValidationFailureWithOptionalLineage({
103
- message: validationMessage(inputValidation.issues),
104
- preserve,
105
- lineage: lineageFromInput(options),
106
- decisionSeed: "input-validation",
107
- });
108
- }
109
- if (isScriptTransformInput(inputValidation.value)) {
110
- return handleScriptTransform({
111
- input: inputValidation.value,
112
- options,
113
- preserve,
114
- });
115
- }
116
- const deterministicInput = inputValidation.value;
117
- const operation = deterministicInput.operation;
118
- const prepared = prepareTransformOperation(operation);
119
- if (!prepared.ok) {
120
- return transformValidationFailureWithOptionalLineage({
121
- message: prepared.message,
122
- preserve,
123
- lineage: lineageFromInput(deterministicInput),
124
- decisionSeed: "operation-validation",
125
- });
126
- }
127
- const source = deterministicInput.source;
128
- const vaultOptions = {};
129
- if (options.vaultRoot !== undefined) {
130
- vaultOptions.root = options.vaultRoot;
131
- }
132
- if (options.vaultRetention !== undefined) {
133
- vaultOptions.retention = options.vaultRetention;
134
- }
135
- const vault = createVault(vaultOptions);
136
- const thresholds = { ...DEFAULT_ROUTER_THRESHOLDS, ...options.thresholds };
137
- let sourceRecord;
138
- let stream;
139
- let sourceText;
140
- try {
141
- sourceRecord = await readVaultRecord(vault, options.sessionId, source.outputId);
142
- } catch (error) {
143
- return transformSourceUnavailableFailureWithOptionalLineage({
144
- message: `Vault source outputId=${source.outputId} could not be found or read: ${errorMessage(error)}`,
145
- preserve,
146
- lineage: lineageFromInput(inputValidation.value),
147
- decisionSeed: "source-record",
148
- });
149
- }
150
- const streamResult = resolveSourceStream(sourceRecord, source.stream);
151
- if (!streamResult.ok) {
152
- return transformValidationFailure({
153
- message: streamResult.message,
154
- preserve,
155
- lineage: lineageForSource(sourceRecord, operation),
156
- decisionSeed: "source-stream",
157
- });
158
- }
159
- stream = streamResult.stream;
160
- try {
161
- sourceText = await readOutputText(vault, options.sessionId, source.outputId, stream);
162
- } catch (error) {
163
- return transformSourceUnavailableFailure({
164
- message: `Vault source outputId=${source.outputId} stream=${stream} could not be read: ${errorMessage(error)}`,
165
- preserve,
166
- lineage: lineageForSource(sourceRecord, operation),
167
- decisionSeed: "source-text",
168
- });
169
- }
170
- let transformed;
171
- try {
172
- transformed = transformText({
173
- text: sourceText,
174
- sourceLabel: `${source.outputId}:${stream}`,
175
- operation,
176
- prepared: prepared.value,
177
- });
178
- } catch (error) {
179
- return transformExecutionFailure({
180
- message: `Transform operation ${operation.kind} failed: ${errorMessage(error)}`,
181
- preserve,
182
- lineage: lineageForSource(sourceRecord, operation),
183
- decisionSeed: "transform-execution",
184
- });
185
- }
186
- const producer = { kind: "transform", name: operation.kind };
187
- const lineage = lineageForSource(sourceRecord, operation);
188
- let record;
189
- try {
190
- record = await storeTextOutput(vault, {
191
- sessionId: options.sessionId,
192
- raw: transformed.text,
193
- sourceKind: "transform",
194
- producer,
195
- lineage,
196
- decisionIds: [
197
- decisionId("transform-store", source.outputId, stream, operation.kind, lineage.operationHash ?? ""),
198
- ],
199
- });
200
- } catch (error) {
201
- return storageFailure({
202
- message: `Transformed output could not be persisted: ${errorMessage(error)}`,
203
- preserve,
204
- lineage,
205
- });
206
- }
207
- const routed = routeTransformedText({
208
- outputId: record.outputId,
209
- text: transformed.text,
210
- preserve,
211
- thresholds,
212
- source: { kind: "vault", outputId: source.outputId, stream },
213
- operationKind: operation.kind,
214
- });
215
- return {
216
- toolStatus: "ok",
217
- decisionId: decisionId("transform", record.outputId, source.outputId, stream, operation.kind, routed.routingStatus),
218
- outputId: record.outputId,
219
- recordId: record.recordId,
220
- preserve,
221
- source: { kind: "vault", outputId: source.outputId, stream },
222
- operation: operationSummary(operation),
223
- producer: record.producer,
224
- persistence: record.persistence,
225
- lineage,
226
- routing: {
227
- status: routed.routingStatus,
228
- route: "transform",
229
- reason: routed.reason,
230
- },
231
- summary: transformed.summary,
232
- evidence: routed.evidence,
233
- recovery: {
234
- how: `Use freeflow_search with source.kind=vault and outputId=${record.outputId}, stream=raw, and an exact lineRange to recover exact transformed content. Source evidence remains outputId=${source.outputId} stream=${stream}.`,
235
- outputId: record.outputId,
236
- },
237
- };
238
- }
239
- async function handleScriptTransform(options) {
240
- const config = effectiveScriptTransformConfig(options.options.scriptTransform);
241
- const limits = effectiveScriptLimits(config, options.input.limits);
242
- const initialLineage = lineageForScriptInput(options.input, limits);
243
- if (!config.enabled) {
244
- return scriptTransformDisabledFailure({
245
- message:
246
- "Script transform is disabled by default. Enable scriptTransform.enabled only after a sandbox adapter has passed capability probes and review. No script code was executed.",
247
- preserve: options.preserve,
248
- lineage: initialLineage,
249
- decisionSeed: "script-transform-disabled",
250
- });
251
- }
252
- const vaultOptions = {};
253
- if (options.options.vaultRoot !== undefined) {
254
- vaultOptions.root = options.options.vaultRoot;
255
- }
256
- if (options.options.vaultRetention !== undefined) {
257
- vaultOptions.retention = options.options.vaultRetention;
258
- }
259
- const vault = createVault(vaultOptions);
260
- const resolved = await resolveScriptSources({
261
- vault,
262
- sessionId: options.options.sessionId,
263
- sources: options.input.sources,
264
- maxInputBytes: limits.maxInputBytes,
265
- operation: options.input.operation,
266
- preserve: options.preserve,
267
- });
268
- if (!resolved.ok) {
269
- return resolved.failure;
270
- }
271
- const adapterSelection = await selectScriptSandboxAdapter(
272
- options.input.operation.language,
273
- config,
274
- options.options.scriptSandboxAdapters ?? [],
275
- );
276
- if (!adapterSelection.ok) {
277
- return transformAdapterUnavailableFailure({
278
- message: `${adapterSelection.status.reason} No script code was executed.`,
279
- preserve: options.preserve,
280
- lineage: lineageForResolvedScriptSources(resolved.sources, options.input, limits),
281
- decisionSeed: "script-transform-adapter-unavailable",
282
- });
283
- }
284
- const execution = await executeScriptWithAdapter({
285
- adapter: adapterSelection.adapter,
286
- operation: options.input.operation,
287
- resolvedSources: resolved.sources,
288
- limits,
289
- config,
290
- });
291
- const lineage = lineageForResolvedScriptSources(resolved.sources, options.input, limits);
292
- if (!execution.ok) {
293
- return transformExecutionFailure({
294
- message: execution.message,
295
- preserve: options.preserve,
296
- lineage,
297
- decisionSeed: "script-transform-execution",
298
- });
299
- }
300
- const producer = { kind: "transform", name: `script:${options.input.operation.language}` };
301
- const vaultOptionsForStore = {};
302
- if (options.options.vaultRoot !== undefined) {
303
- vaultOptionsForStore.root = options.options.vaultRoot;
304
- }
305
- if (options.options.vaultRetention !== undefined) {
306
- vaultOptionsForStore.retention = options.options.vaultRetention;
307
- }
308
- const storeVault = createVault(vaultOptionsForStore);
309
- let record;
310
- try {
311
- record = await storeTextOutput(storeVault, {
312
- sessionId: options.options.sessionId,
313
- raw: execution.result.stdout,
314
- sourceKind: "transform",
315
- producer,
316
- lineage,
317
- decisionIds: [
318
- decisionId("script-transform-store", options.input.operation.language, lineage.operationHash ?? ""),
319
- ],
320
- });
321
- } catch (error) {
322
- return storageFailure({
323
- message: `Script-transformed output could not be persisted: ${errorMessage(error)}`,
324
- preserve: options.preserve,
325
- lineage,
326
- });
327
- }
328
- const thresholds = { ...DEFAULT_ROUTER_THRESHOLDS, ...options.options.thresholds };
329
- const primarySource = primaryScriptSourceRef(resolved.sources, options.input.sources);
330
- const routed = routeTransformedText({
331
- outputId: record.outputId,
332
- text: execution.result.stdout,
333
- preserve: options.preserve,
334
- thresholds,
335
- source: primarySource,
336
- operationKind: "script",
337
- });
338
- return {
339
- toolStatus: "ok",
340
- decisionId: decisionId("script-transform", record.outputId, options.input.operation.language, routed.routingStatus),
341
- outputId: record.outputId,
342
- recordId: record.recordId,
343
- preserve: options.preserve,
344
- source: primarySource,
345
- operation: operationSummary(options.input.operation),
346
- producer: record.producer,
347
- persistence: record.persistence,
348
- lineage,
349
- routing: {
350
- status: routed.routingStatus,
351
- route: "transform",
352
- reason: routed.reason,
353
- },
354
- summary: `Script transform ${options.input.operation.language} completed with ${byteLength(execution.result.stdout)} stdout bytes and ${byteLength(execution.result.stderr)} stderr bytes.`,
355
- evidence: routed.evidence,
356
- recovery: {
357
- how: `Use freeflow_search with source.kind=vault and outputId=${record.outputId}, stream=raw, and an exact lineRange to recover exact script-transformed content. Source evidence remains linked in lineage.sourceOutputIds.`,
358
- outputId: record.outputId,
359
- },
360
- };
361
- }
362
- function primaryScriptSourceRef(resolvedSources, inputSources) {
363
- const resolved = resolvedSources[0];
364
- if (resolved) {
365
- return { kind: "vault", outputId: resolved.outputId, stream: resolved.stream };
366
- }
367
- const input = inputSources[0];
368
- if (input?.stream !== undefined) {
369
- return { kind: "vault", outputId: input.outputId, stream: input.stream };
370
- }
371
- return { kind: "vault", outputId: input?.outputId ?? "unknown" };
372
- }
373
- export async function executeSandboxedScriptOperation(options) {
374
- const config = effectiveScriptTransformConfig(options.scriptTransform);
375
- const limits = effectiveScriptLimits(config, options.limits);
376
- const operation = scriptOperationSummary(options.operation);
377
- if (!config.enabled) {
378
- return {
379
- ok: false,
380
- failureKind: "script_transform_disabled",
381
- executionStatus: "unavailable",
382
- message:
383
- "Script transform is disabled by default. Enable scriptTransform.enabled only after a sandbox adapter has passed capability probes and review. No script code was executed.",
384
- limits,
385
- operation,
386
- };
387
- }
388
- const adapterSelection = await selectScriptSandboxAdapter(
389
- options.operation.language,
390
- config,
391
- options.scriptSandboxAdapters ?? [],
392
- );
393
- if (!adapterSelection.ok) {
394
- const failure = {
395
- ok: false,
396
- failureKind: "adapter_unavailable",
397
- executionStatus: "unavailable",
398
- message: `${adapterSelection.status.reason} No script code was executed.`,
399
- limits,
400
- operation,
401
- failedProofs: adapterSelection.status.failedProofs,
402
- };
403
- if (adapterSelection.status.adapterId !== undefined) {
404
- failure.adapterId = adapterSelection.status.adapterId;
405
- }
406
- if (adapterSelection.status.adapterVersion !== undefined) {
407
- failure.adapterVersion = adapterSelection.status.adapterVersion;
408
- }
409
- return failure;
410
- }
411
- const execution = await runScriptAdapter({
412
- adapter: adapterSelection.adapter,
413
- operation: options.operation,
414
- resolvedSources: [],
415
- limits,
416
- config,
417
- });
418
- if (!execution.ok) {
419
- return {
420
- ok: false,
421
- failureKind: "transform_execution_failure",
422
- executionStatus: "failed",
423
- message: execution.message,
424
- limits,
425
- operation,
426
- adapterId: adapterSelection.adapter.id,
427
- adapterVersion: adapterSelection.adapter.version,
428
- };
429
- }
430
- const success = {
431
- ok: true,
432
- result: execution.result,
433
- limits,
434
- operation,
435
- adapterId: adapterSelection.adapter.id,
436
- adapterVersion: adapterSelection.adapter.version,
437
- };
438
- if (adapterSelection.status.runtime !== undefined) {
439
- success.runtime = adapterSelection.status.runtime;
440
- }
441
- return success;
442
- }
443
- async function executeScriptWithAdapter(options) {
444
- const execution = await runScriptAdapter(options);
445
- if (!execution.ok) {
446
- return execution;
447
- }
448
- const stdoutBytes = byteLength(execution.result.stdout ?? "");
449
- const stderrBytes = byteLength(execution.result.stderr ?? "");
450
- if (execution.result.status !== "success") {
451
- const detail = execution.result.reason ? ` ${execution.result.reason}` : "";
452
- return {
453
- ok: false,
454
- message: `Script transform ${options.operation.language} ${execution.result.status}.${detail} stdoutBytes=${stdoutBytes} stderrBytes=${stderrBytes}.`,
455
- };
456
- }
457
- return { ok: true, result: execution.result };
458
- }
459
- async function runScriptAdapter(options) {
460
- const tempRoot = await mkdtemp(join(tmpdir(), "freeflow-script-transform-"));
461
- const inputDir = join(tempRoot, "input");
462
- const workDir = join(tempRoot, "work");
463
- const outputDir = join(tempRoot, "output");
464
- try {
465
- await mkdir(inputDir, { recursive: true });
466
- await mkdir(workDir, { recursive: true });
467
- await mkdir(outputDir, { recursive: true });
468
- const mounts = [];
469
- for (const source of options.resolvedSources) {
470
- const path = join(inputDir, `${source.alias}.txt`);
471
- await writeFile(path, source.text, "utf8");
472
- mounts.push({ alias: source.alias, path, bytes: source.bytes, sha256: source.textSha256 });
473
- }
474
- await writeFile(
475
- join(inputDir, "manifest.json"),
476
- JSON.stringify(
477
- {
478
- schemaVersion: 1,
479
- sources: options.resolvedSources.map((source) => ({
480
- alias: source.alias,
481
- outputId: source.outputId,
482
- stream: source.stream,
483
- bytes: source.bytes,
484
- sha256: source.textSha256,
485
- })),
486
- },
487
- null,
488
- 2,
489
- ),
490
- "utf8",
491
- );
492
- let result;
493
- try {
494
- result = await options.adapter.execute({
495
- language: options.operation.language,
496
- code: options.operation.code,
497
- inputDir,
498
- workDir,
499
- outputDir,
500
- sources: mounts,
501
- limits: options.limits,
502
- network: options.config.network,
503
- });
504
- } catch (error) {
505
- return {
506
- ok: false,
507
- message: `Script transform adapter ${options.adapter.id} threw before returning a result: ${errorMessage(error)}`,
508
- };
509
- }
510
- const stdoutBytes = byteLength(result.stdout ?? "");
511
- const stderrBytes = byteLength(result.stderr ?? "");
512
- if (stdoutBytes + stderrBytes > options.limits.maxOutputBytes) {
513
- return {
514
- ok: false,
515
- message: `Script transform output bytes ${stdoutBytes + stderrBytes} exceed maxOutputBytes ${options.limits.maxOutputBytes}.`,
516
- };
517
- }
518
- return { ok: true, result };
519
- } finally {
520
- await rm(tempRoot, { recursive: true, force: true });
521
- }
522
- }
523
- async function resolveScriptSources(options) {
524
- const resolved = [];
525
- let totalBytes = 0;
526
- for (const source of options.sources) {
527
- let record;
528
- try {
529
- record = await readVaultRecord(options.vault, options.sessionId, source.outputId);
530
- } catch (error) {
531
- return {
532
- ok: false,
533
- failure: transformSourceUnavailableFailure({
534
- message: `Script transform source alias=${source.alias} outputId=${source.outputId} could not be found or read: ${errorMessage(error)}`,
535
- preserve: options.preserve,
536
- lineage: lineageForScriptInput(
537
- { sources: [...options.sources], operation: options.operation },
538
- { maxInputBytes: options.maxInputBytes },
539
- ),
540
- decisionSeed: "script-source-record",
541
- }),
542
- };
543
- }
544
- const streamResult = resolveSourceStream(record, source.stream);
545
- if (!streamResult.ok) {
546
- return {
547
- ok: false,
548
- failure: transformValidationFailure({
549
- message: `Script transform source alias=${source.alias} outputId=${source.outputId} stream is invalid: ${streamResult.message}`,
550
- preserve: options.preserve,
551
- lineage: lineageForSource(record, options.operation),
552
- decisionSeed: "script-source-stream",
553
- }),
554
- };
555
- }
556
- let text;
557
- try {
558
- text = await readOutputText(options.vault, options.sessionId, source.outputId, streamResult.stream);
559
- } catch (error) {
560
- return {
561
- ok: false,
562
- failure: transformSourceUnavailableFailure({
563
- message: `Script transform source alias=${source.alias} outputId=${source.outputId} stream=${streamResult.stream} could not be read: ${errorMessage(error)}`,
564
- preserve: options.preserve,
565
- lineage: lineageForSource(record, options.operation),
566
- decisionSeed: "script-source-text",
567
- }),
568
- };
569
- }
570
- const bytes = byteLength(text);
571
- totalBytes += bytes;
572
- if (totalBytes > options.maxInputBytes) {
573
- return {
574
- ok: false,
575
- failure: transformValidationFailure({
576
- message: `Script transform input bytes ${totalBytes} exceed maxInputBytes ${options.maxInputBytes}.`,
577
- preserve: options.preserve,
578
- lineage: lineageForSource(record, options.operation),
579
- decisionSeed: "script-source-size",
580
- }),
581
- };
582
- }
583
- resolved.push({
584
- alias: source.alias,
585
- outputId: source.outputId,
586
- recordId: record.recordId,
587
- stream: streamResult.stream,
588
- bytes,
589
- contentHashSha256: record.contentHashSha256,
590
- textSha256: hashText(text),
591
- text,
592
- });
593
- }
594
- return { ok: true, sources: resolved };
595
- }
596
- function validateTransformSource(value, path, issues) {
597
- if (!isRecord(value)) {
598
- issues.push({ path, message: "Expected transform source object." });
599
- return;
600
- }
601
- if (value.kind !== "vault") {
602
- issues.push({ path: `${path}.kind`, message: "Slice 5A supports only vault transform sources." });
603
- return;
604
- }
605
- if (typeof value.outputId !== "string" || value.outputId.length === 0) {
606
- issues.push({ path: `${path}.outputId`, message: "Expected non-empty vault outputId." });
607
- }
608
- if (value.stream !== undefined && !isOutputStream(value.stream)) {
609
- issues.push({ path: `${path}.stream`, message: "Expected a known output stream." });
610
- }
611
- }
612
- function validateTransformOperation(value, path, issues) {
613
- if (!isRecord(value)) {
614
- issues.push({ path, message: "Expected transform operation object." });
615
- return;
616
- }
617
- if (typeof value.kind !== "string" || !OPERATION_KINDS.has(value.kind)) {
618
- issues.push({ path: `${path}.kind`, message: "Expected a supported transform operation kind." });
619
- return;
620
- }
621
- if (value.kind === "script") {
622
- validateScriptOperation(value, path, issues);
623
- return;
624
- }
625
- if (value.kind === "jsonExtract") {
626
- validateJsonExtractOperation(value, path, issues);
627
- return;
628
- }
629
- if (value.kind === "dedupe") {
630
- validateDedupeOperation(value, path, issues);
631
- return;
632
- }
633
- if (value.kind === "topN") {
634
- validateTopNOperation(value, path, issues);
635
- return;
636
- }
637
- if (value.kind === "extractUrls" || value.kind === "extractCitations") {
638
- validateExtractOperation(value, path, issues);
639
- return;
640
- }
641
- if (value.kind === "lineStats" || value.kind === "sizeStats") {
642
- validateStatsOperation(value, path, issues);
643
- return;
644
- }
645
- if (typeof value.pattern !== "string" || value.pattern.length === 0) {
646
- issues.push({ path: `${path}.pattern`, message: "Expected a non-empty regex pattern string." });
647
- }
648
- validateRegexFlags(value.flags, `${path}.flags`, issues);
649
- if (value.kind === "regexFilter") {
650
- if (value.contextLines !== undefined) {
651
- validateIntegerRange(value.contextLines, `${path}.contextLines`, 0, MAX_CONTEXT_LINES, issues);
652
- }
653
- if (value.maxMatches !== undefined) {
654
- validateIntegerRange(value.maxMatches, `${path}.maxMatches`, 1, MAX_MATCHES, issues);
655
- }
656
- return;
657
- }
658
- if (value.kind === "groupByRegex") {
659
- validateGroupSelector(value.group, `${path}.group`, issues);
660
- if (value.maxGroups !== undefined) {
661
- validateIntegerRange(value.maxGroups, `${path}.maxGroups`, 1, MAX_GROUPS, issues);
662
- }
663
- if (value.maxLinesPerGroup !== undefined) {
664
- validateIntegerRange(value.maxLinesPerGroup, `${path}.maxLinesPerGroup`, 1, MAX_LINES_PER_GROUP, issues);
665
- }
666
- return;
667
- }
668
- for (const key of ["contextLines", "maxMatches", "group", "maxGroups", "maxLinesPerGroup"]) {
669
- if (value[key] !== undefined) {
670
- issues.push({ path: `${path}.${key}`, message: `Operation ${value.kind} does not accept ${key}.` });
671
- }
672
- }
673
- }
674
- function validateScriptOperation(value, path, issues) {
675
- if (!isStringIn(value.language, SCRIPT_TRANSFORM_LANGUAGES)) {
676
- issues.push({
677
- path: `${path}.language`,
678
- message: `Expected script language ${SCRIPT_TRANSFORM_LANGUAGES.join(", ")}.`,
679
- });
680
- }
681
- if (typeof value.code !== "string" || value.code.length === 0) {
682
- issues.push({ path: `${path}.code`, message: "Expected non-empty script code string." });
683
- } else if (byteLength(value.code) > MAX_SCRIPT_CODE_BYTES) {
684
- issues.push({ path: `${path}.code`, message: `Script code must be at most ${MAX_SCRIPT_CODE_BYTES} bytes.` });
685
- }
686
- if (value.label !== undefined && (typeof value.label !== "string" || value.label.length === 0)) {
687
- issues.push({ path: `${path}.label`, message: "Expected non-empty script label string when present." });
688
- }
689
- }
690
- function validateScriptTransformSources(value, path, issues) {
691
- if (!Array.isArray(value) || value.length === 0) {
692
- issues.push({ path, message: "Expected one or more script transform vault sources." });
693
- return;
694
- }
695
- const aliases = new Set();
696
- value.forEach((source, index) => {
697
- const sourcePath = `${path}[${index}]`;
698
- validateTransformSource(source, sourcePath, issues);
699
- if (!isRecord(source)) {
700
- return;
701
- }
702
- if (typeof source.alias !== "string" || !SCRIPT_ALIAS_PATTERN.test(source.alias)) {
703
- issues.push({ path: `${sourcePath}.alias`, message: "Expected alias matching ^[A-Za-z][A-Za-z0-9_-]{0,63}$." });
704
- return;
705
- }
706
- if (aliases.has(source.alias)) {
707
- issues.push({ path: `${sourcePath}.alias`, message: `Duplicate script source alias ${source.alias}.` });
708
- }
709
- aliases.add(source.alias);
710
- });
711
- }
712
- function validateScriptTransformLimits(value, path, issues) {
713
- if (value === undefined) {
714
- return;
715
- }
716
- if (!isRecord(value)) {
717
- issues.push({ path, message: "Expected script transform limits object." });
718
- return;
719
- }
720
- validateOptionalIntegerRange(value.timeoutMs, `${path}.timeoutMs`, 1, MAX_SCRIPT_TRANSFORM_LIMITS.timeoutMs, issues);
721
- validateOptionalIntegerRange(
722
- value.maxInputBytes,
723
- `${path}.maxInputBytes`,
724
- 1,
725
- MAX_SCRIPT_TRANSFORM_LIMITS.maxInputBytes,
726
- issues,
727
- );
728
- validateOptionalIntegerRange(
729
- value.maxOutputBytes,
730
- `${path}.maxOutputBytes`,
731
- 1,
732
- MAX_SCRIPT_TRANSFORM_LIMITS.maxOutputBytes,
733
- issues,
734
- );
735
- }
736
- function validateOptionalIntegerRange(value, path, min, max, issues) {
737
- if (value === undefined) {
738
- return;
739
- }
740
- validateIntegerRange(value, path, min, max, issues);
741
- }
742
- function validateJsonExtractOperation(value, path, issues) {
743
- const hasPointer = value.pointer !== undefined;
744
- const hasPath = value.path !== undefined;
745
- if (hasPointer === hasPath) {
746
- issues.push({ path, message: "Expected exactly one JSON selector: pointer or path." });
747
- }
748
- if (value.pointer !== undefined) {
749
- if (typeof value.pointer !== "string") {
750
- issues.push({ path: `${path}.pointer`, message: "Expected JSON pointer string." });
751
- } else {
752
- const pointer = parseJsonPointer(value.pointer);
753
- if (!pointer.ok) {
754
- issues.push({ path: `${path}.pointer`, message: `Invalid JSON pointer: ${pointer.message}` });
755
- }
756
- }
757
- }
758
- if (value.path !== undefined) {
759
- if (typeof value.path !== "string") {
760
- issues.push({ path: `${path}.path`, message: "Expected JSON path string." });
761
- } else {
762
- const parsedPath = parseJsonPath(value.path);
763
- if (!parsedPath.ok) {
764
- issues.push({ path: `${path}.path`, message: `Invalid JSON path: ${parsedPath.message}` });
765
- }
766
- }
767
- }
768
- for (const key of ["pattern", "flags", "contextLines", "maxMatches"]) {
769
- if (value[key] !== undefined) {
770
- issues.push({ path: `${path}.${key}`, message: `Operation jsonExtract does not accept ${key}.` });
771
- }
772
- }
773
- }
774
- function validateDedupeOperation(value, path, issues) {
775
- if (value.trim !== undefined && typeof value.trim !== "boolean") {
776
- issues.push({ path: `${path}.trim`, message: "Expected boolean when present." });
777
- }
778
- if (value.caseSensitive !== undefined && typeof value.caseSensitive !== "boolean") {
779
- issues.push({ path: `${path}.caseSensitive`, message: "Expected boolean when present." });
780
- }
781
- if (value.maxLines !== undefined) {
782
- validateIntegerRange(value.maxLines, `${path}.maxLines`, 1, MAX_DEDUPE_LINES, issues);
783
- }
784
- for (const key of [
785
- "pattern",
786
- "flags",
787
- "contextLines",
788
- "maxMatches",
789
- "group",
790
- "maxGroups",
791
- "maxLinesPerGroup",
792
- "limit",
793
- "sort",
794
- "order",
795
- ]) {
796
- if (value[key] !== undefined) {
797
- issues.push({ path: `${path}.${key}`, message: `Operation dedupe does not accept ${key}.` });
798
- }
799
- }
800
- }
801
- function validateTopNOperation(value, path, issues) {
802
- validateIntegerRange(value.limit, `${path}.limit`, 1, MAX_TOP_N_LIMIT, issues);
803
- const hasPattern = value.pattern !== undefined;
804
- if (hasPattern) {
805
- if (typeof value.pattern !== "string" || value.pattern.length === 0) {
806
- issues.push({ path: `${path}.pattern`, message: "Expected a non-empty regex pattern string." });
807
- }
808
- validateRegexFlags(value.flags, `${path}.flags`, issues);
809
- validateGroupSelector(value.group, `${path}.group`, issues);
810
- } else {
811
- if (value.flags !== undefined) {
812
- issues.push({ path: `${path}.flags`, message: "topN flags require a pattern." });
813
- }
814
- if (value.group !== undefined) {
815
- issues.push({ path: `${path}.group`, message: "topN group requires a pattern." });
816
- }
817
- }
818
- if (value.sort !== undefined && value.sort !== "text" && value.sort !== "numeric") {
819
- issues.push({ path: `${path}.sort`, message: "Expected sort text or numeric." });
820
- }
821
- if (value.order !== undefined && value.order !== "asc" && value.order !== "desc") {
822
- issues.push({ path: `${path}.order`, message: "Expected order asc or desc." });
823
- }
824
- for (const key of [
825
- "contextLines",
826
- "maxMatches",
827
- "maxGroups",
828
- "maxLinesPerGroup",
829
- "trim",
830
- "caseSensitive",
831
- "maxLines",
832
- ]) {
833
- if (value[key] !== undefined) {
834
- issues.push({ path: `${path}.${key}`, message: `Operation topN does not accept ${key}.` });
835
- }
836
- }
837
- }
838
- function validateExtractOperation(value, path, issues) {
839
- if (value.maxMatches !== undefined) {
840
- validateIntegerRange(value.maxMatches, `${path}.maxMatches`, 1, MAX_EXTRACT_MATCHES, issues);
841
- }
842
- if (value.kind === "extractUrls" && value.dedupe !== undefined && typeof value.dedupe !== "boolean") {
843
- issues.push({ path: `${path}.dedupe`, message: "Expected boolean when present." });
844
- }
845
- if (value.kind === "extractCitations" && value.dedupe !== undefined) {
846
- issues.push({ path: `${path}.dedupe`, message: "Operation extractCitations does not accept dedupe." });
847
- }
848
- for (const key of [
849
- "pattern",
850
- "flags",
851
- "contextLines",
852
- "group",
853
- "maxGroups",
854
- "maxLinesPerGroup",
855
- "limit",
856
- "sort",
857
- "order",
858
- "trim",
859
- "caseSensitive",
860
- "maxLines",
861
- ]) {
862
- if (value[key] !== undefined) {
863
- issues.push({ path: `${path}.${key}`, message: `Operation ${value.kind} does not accept ${key}.` });
864
- }
865
- }
866
- }
867
- function validateStatsOperation(value, path, issues) {
868
- for (const key of [
869
- "pattern",
870
- "flags",
871
- "contextLines",
872
- "maxMatches",
873
- "group",
874
- "maxGroups",
875
- "maxLinesPerGroup",
876
- "limit",
877
- "sort",
878
- "order",
879
- "trim",
880
- "caseSensitive",
881
- "maxLines",
882
- "dedupe",
883
- ]) {
884
- if (value[key] !== undefined) {
885
- issues.push({ path: `${path}.${key}`, message: `Operation ${value.kind} does not accept ${key}.` });
886
- }
887
- }
888
- }
889
- function validateGroupSelector(value, path, issues) {
890
- if (value === undefined) {
891
- return;
892
- }
893
- if (typeof value === "number") {
894
- if (!Number.isInteger(value) || value < 0) {
895
- issues.push({ path, message: "Expected a non-negative integer group index." });
896
- }
897
- return;
898
- }
899
- if (typeof value === "string" && value.length > 0) {
900
- return;
901
- }
902
- issues.push({ path, message: "Expected a non-empty string group name or non-negative integer group index." });
903
- }
904
- function validateRegexFlags(value, path, issues) {
905
- if (value === undefined) {
906
- return;
907
- }
908
- if (typeof value !== "string") {
909
- issues.push({ path, message: "Expected regex flags string when present." });
910
- return;
911
- }
912
- const seen = new Set();
913
- for (const flag of value) {
914
- if (!REGEX_FLAGS.has(flag)) {
915
- issues.push({ path, message: "Expected regex flags to contain only g, i, m, s, or u." });
916
- return;
917
- }
918
- if (seen.has(flag)) {
919
- issues.push({ path, message: "Expected regex flags without duplicates." });
920
- return;
921
- }
922
- seen.add(flag);
923
- }
924
- }
925
- function validateIntegerRange(value, path, min, max, issues) {
926
- if (!Number.isInteger(value) || value < min || value > max) {
927
- issues.push({ path, message: `Expected integer from ${min} to ${max}.` });
928
- }
929
- }
930
- function prepareTransformOperation(operation) {
931
- if (operation.kind === "jsonExtract") {
932
- const preparedJson = prepareJsonExtractOperation(operation);
933
- if (!preparedJson.ok) {
934
- return preparedJson;
935
- }
936
- return { ok: true, value: { kind: "json", value: preparedJson.value } };
937
- }
938
- if (
939
- operation.kind === "dedupe" ||
940
- operation.kind === "extractUrls" ||
941
- operation.kind === "extractCitations" ||
942
- operation.kind === "lineStats" ||
943
- operation.kind === "sizeStats" ||
944
- (operation.kind === "topN" && operation.pattern === undefined)
945
- ) {
946
- return { ok: true, value: { kind: "none" } };
947
- }
948
- const compiledRegex = compileRegexOperation(operation);
949
- if (!compiledRegex.ok) {
950
- return compiledRegex;
951
- }
952
- return { ok: true, value: { kind: "regex", value: compiledRegex.value } };
953
- }
954
- function compileRegexOperation(operation) {
955
- const flags = normalizeRegexFlags(operation.flags);
956
- let regex;
957
- try {
958
- regex = new RegExp(operation.pattern, ensureGlobalFlag(flags));
959
- } catch (error) {
960
- return { ok: false, message: `Invalid regex pattern for ${operation.kind}: ${errorMessage(error)}` };
961
- }
962
- try {
963
- const zeroWidthCheck = new RegExp(operation.pattern, flags);
964
- if (zeroWidthCheck.test("")) {
965
- return {
966
- ok: false,
967
- message: `Invalid regex pattern for ${operation.kind}: patterns that match empty strings are not supported.`,
968
- };
969
- }
970
- } catch (error) {
971
- return { ok: false, message: `Invalid regex pattern for ${operation.kind}: ${errorMessage(error)}` };
972
- }
973
- return {
974
- ok: true,
975
- value: {
976
- regex,
977
- displayPattern: operation.pattern,
978
- flags,
979
- },
980
- };
981
- }
982
- function prepareJsonExtractOperation(operation) {
983
- if (operation.pointer !== undefined) {
984
- const pointer = parseJsonPointer(operation.pointer);
985
- if (!pointer.ok) {
986
- return { ok: false, message: `Invalid JSON pointer: ${pointer.message}` };
987
- }
988
- return {
989
- ok: true,
990
- value: {
991
- selectorKind: "pointer",
992
- selector: operation.pointer,
993
- segments: pointer.segments,
994
- },
995
- };
996
- }
997
- if (operation.path !== undefined) {
998
- const path = parseJsonPath(operation.path);
999
- if (!path.ok) {
1000
- return { ok: false, message: `Invalid JSON path: ${path.message}` };
1001
- }
1002
- return {
1003
- ok: true,
1004
- value: {
1005
- selectorKind: "path",
1006
- selector: operation.path,
1007
- segments: path.segments,
1008
- },
1009
- };
1010
- }
1011
- return { ok: false, message: "jsonExtract requires exactly one JSON selector: pointer or path." };
1012
- }
1013
- function transformText(options) {
1014
- if (options.operation.kind === "jsonExtract") {
1015
- if (options.prepared.kind !== "json") {
1016
- throw new Error("jsonExtract operation was not prepared with a JSON selector.");
1017
- }
1018
- return transformJsonExtract({
1019
- text: options.text,
1020
- sourceLabel: options.sourceLabel,
1021
- operation: options.operation,
1022
- prepared: options.prepared.value,
1023
- });
1024
- }
1025
- if (options.operation.kind === "dedupe") {
1026
- return transformDedupe({
1027
- text: options.text,
1028
- sourceLabel: options.sourceLabel,
1029
- operation: options.operation,
1030
- });
1031
- }
1032
- if (options.operation.kind === "topN") {
1033
- const topNOptions = {
1034
- text: options.text,
1035
- sourceLabel: options.sourceLabel,
1036
- operation: options.operation,
1037
- };
1038
- if (options.prepared.kind === "regex") {
1039
- topNOptions.compiled = options.prepared.value;
1040
- }
1041
- return transformTopN(topNOptions);
1042
- }
1043
- if (options.operation.kind === "extractUrls") {
1044
- return transformExtractUrls({ text: options.text, sourceLabel: options.sourceLabel, operation: options.operation });
1045
- }
1046
- if (options.operation.kind === "extractCitations") {
1047
- return transformExtractCitations({
1048
- text: options.text,
1049
- sourceLabel: options.sourceLabel,
1050
- operation: options.operation,
1051
- });
1052
- }
1053
- if (options.operation.kind === "lineStats") {
1054
- return transformLineStats({ text: options.text, sourceLabel: options.sourceLabel });
1055
- }
1056
- if (options.operation.kind === "sizeStats") {
1057
- return transformSizeStats({ text: options.text, sourceLabel: options.sourceLabel });
1058
- }
1059
- if (options.prepared.kind !== "regex") {
1060
- throw new Error(`${options.operation.kind} operation was not prepared with a regex.`);
1061
- }
1062
- if (options.operation.kind === "regexFilter") {
1063
- return transformRegexFilter({
1064
- text: options.text,
1065
- sourceLabel: options.sourceLabel,
1066
- operation: options.operation,
1067
- compiled: options.prepared.value,
1068
- });
1069
- }
1070
- if (options.operation.kind === "groupByRegex") {
1071
- return transformGroupByRegex({
1072
- text: options.text,
1073
- sourceLabel: options.sourceLabel,
1074
- operation: options.operation,
1075
- compiled: options.prepared.value,
1076
- });
1077
- }
1078
- return transformCountMatches({
1079
- text: options.text,
1080
- sourceLabel: options.sourceLabel,
1081
- operation: options.operation,
1082
- compiled: options.prepared.value,
1083
- });
1084
- }
1085
- function transformRegexFilter(options) {
1086
- const lines = splitLines(options.text);
1087
- const contextLines = options.operation.contextLines ?? DEFAULT_CONTEXT_LINES;
1088
- const maxMatches = options.operation.maxMatches ?? DEFAULT_MAX_MATCHES;
1089
- const stats = collectMatches(lines, options.compiled.regex, maxMatches);
1090
- const windows = mergeLineWindows(
1091
- stats.matchedLineNumbers.map((lineNumber) => ({
1092
- start: Math.max(1, lineNumber - contextLines),
1093
- end: Math.min(lines.length, lineNumber + contextLines),
1094
- })),
1095
- );
1096
- const parts = [
1097
- "# freeflow_search action=transform regexFilter",
1098
- `source: ${options.sourceLabel}`,
1099
- `pattern: ${formatPattern(options.compiled)}`,
1100
- `contextLines: ${contextLines}`,
1101
- `maxMatches: ${maxMatches}`,
1102
- `matches: ${stats.matches}`,
1103
- `matchedLines: ${stats.matchedLines}`,
1104
- `truncated: ${stats.truncated}`,
1105
- ];
1106
- for (const window of windows) {
1107
- parts.push("", `@@ source lines ${window.start}-${window.end} @@`);
1108
- for (let lineNumber = window.start; lineNumber <= window.end; lineNumber += 1) {
1109
- parts.push(`${lineNumber}| ${lines[lineNumber - 1] ?? ""}`);
1110
- }
1111
- }
1112
- return {
1113
- text: `${parts.join("\n")}\n`,
1114
- summary: `Transformed regexFilter from vaulted ${sourceStreamLabel(options.sourceLabel)} output: ${stats.matches} match(es) across ${stats.matchedLines} line(s).`,
1115
- stats,
1116
- };
1117
- }
1118
- function transformCountMatches(options) {
1119
- const lines = splitLines(options.text);
1120
- const stats = collectMatches(lines, options.compiled.regex);
1121
- const text = [
1122
- "# freeflow_search action=transform countMatches",
1123
- `source: ${options.sourceLabel}`,
1124
- `pattern: ${formatPattern(options.compiled)}`,
1125
- `matches: ${stats.matches}`,
1126
- `matchedLines: ${stats.matchedLines}`,
1127
- "",
1128
- ].join("\n");
1129
- return {
1130
- text,
1131
- summary: `Transformed countMatches from vaulted ${sourceStreamLabel(options.sourceLabel)} output: ${stats.matches} match(es) across ${stats.matchedLines} line(s).`,
1132
- stats,
1133
- };
1134
- }
1135
- function transformGroupByRegex(options) {
1136
- const lines = splitLines(options.text);
1137
- const groupSelector = options.operation.group ?? 1;
1138
- const maxGroups = options.operation.maxGroups ?? DEFAULT_MAX_GROUPS;
1139
- const maxLinesPerGroup = options.operation.maxLinesPerGroup ?? DEFAULT_MAX_LINES_PER_GROUP;
1140
- const groups = new Map();
1141
- const allGroupKeys = new Set();
1142
- let matchedLines = 0;
1143
- let truncated = false;
1144
- for (let index = 0; index < lines.length; index += 1) {
1145
- const line = lines[index] ?? "";
1146
- options.compiled.regex.lastIndex = 0;
1147
- const match = options.compiled.regex.exec(line);
1148
- if (!match) {
1149
- continue;
1150
- }
1151
- if (match[0].length === 0) {
1152
- throw new Error("Regex patterns that produce zero-width matches are not supported.");
1153
- }
1154
- matchedLines += 1;
1155
- const groupKey = matchGroupValue(match, groupSelector, index + 1);
1156
- allGroupKeys.add(groupKey);
1157
- let group = groups.get(groupKey);
1158
- if (!group) {
1159
- if (groups.size >= maxGroups) {
1160
- truncated = true;
1161
- continue;
1162
- }
1163
- group = { count: 0, entries: [] };
1164
- groups.set(groupKey, group);
1165
- }
1166
- group.count += 1;
1167
- if (group.entries.length < maxLinesPerGroup) {
1168
- group.entries.push({ lineNumber: index + 1, line });
1169
- } else {
1170
- truncated = true;
1171
- }
1172
- }
1173
- const parts = [
1174
- "# freeflow_search action=transform groupByRegex",
1175
- `source: ${options.sourceLabel}`,
1176
- `pattern: ${formatPattern(options.compiled)}`,
1177
- `group: ${String(groupSelector)}`,
1178
- `maxGroups: ${maxGroups}`,
1179
- `maxLinesPerGroup: ${maxLinesPerGroup}`,
1180
- `groups: ${allGroupKeys.size}`,
1181
- `returnedGroups: ${groups.size}`,
1182
- `matchedLines: ${matchedLines}`,
1183
- `truncated: ${truncated}`,
1184
- ];
1185
- for (const [groupKey, group] of groups.entries()) {
1186
- parts.push("", `## group: ${groupKey}`, `count: ${group.count}`);
1187
- for (const entry of group.entries) {
1188
- parts.push(`${entry.lineNumber}| ${entry.line}`);
1189
- }
1190
- }
1191
- return {
1192
- text: `${parts.join("\n")}\n`,
1193
- summary: `Transformed groupByRegex from vaulted ${sourceStreamLabel(options.sourceLabel)} output: ${allGroupKeys.size} group(s), ${matchedLines} matched line(s).`,
1194
- stats: {
1195
- matches: matchedLines,
1196
- matchedLines,
1197
- matchedLineNumbers: [],
1198
- truncated,
1199
- },
1200
- };
1201
- }
1202
- function transformDedupe(options) {
1203
- const lines = splitLines(options.text);
1204
- const trim = options.operation.trim ?? false;
1205
- const caseSensitive = options.operation.caseSensitive ?? true;
1206
- const maxLines = options.operation.maxLines ?? DEFAULT_MAX_DEDUPE_LINES;
1207
- const seen = new Set();
1208
- const uniqueEntries = [];
1209
- let duplicatesRemoved = 0;
1210
- let truncated = false;
1211
- for (let index = 0; index < lines.length; index += 1) {
1212
- const line = lines[index] ?? "";
1213
- const displayLine = trim ? line.trim() : line;
1214
- const key = caseSensitive ? displayLine : displayLine.toLowerCase();
1215
- if (seen.has(key)) {
1216
- duplicatesRemoved += 1;
1217
- continue;
1218
- }
1219
- seen.add(key);
1220
- if (uniqueEntries.length < maxLines) {
1221
- uniqueEntries.push({ lineNumber: index + 1, line: displayLine });
1222
- } else {
1223
- truncated = true;
1224
- }
1225
- }
1226
- const parts = [
1227
- "# freeflow_search action=transform dedupe",
1228
- `source: ${options.sourceLabel}`,
1229
- `trim: ${trim}`,
1230
- `caseSensitive: ${caseSensitive}`,
1231
- `maxLines: ${maxLines}`,
1232
- `inputLines: ${lines.length}`,
1233
- `uniqueLines: ${seen.size}`,
1234
- `returnedLines: ${uniqueEntries.length}`,
1235
- `duplicatesRemoved: ${duplicatesRemoved}`,
1236
- `truncated: ${truncated}`,
1237
- "",
1238
- ];
1239
- for (const entry of uniqueEntries) {
1240
- parts.push(`${entry.lineNumber}| ${entry.line}`);
1241
- }
1242
- return {
1243
- text: `${parts.join("\n")}\n`,
1244
- summary: `Transformed dedupe from vaulted ${sourceStreamLabel(options.sourceLabel)} output: ${seen.size} unique line(s), ${duplicatesRemoved} duplicate line(s) removed.`,
1245
- stats: {
1246
- matches: seen.size,
1247
- matchedLines: seen.size,
1248
- matchedLineNumbers: uniqueEntries.map((entry) => entry.lineNumber),
1249
- truncated,
1250
- },
1251
- };
1252
- }
1253
- function transformTopN(options) {
1254
- const lines = splitLines(options.text);
1255
- const sort = options.operation.sort ?? "text";
1256
- const order = options.operation.order ?? "asc";
1257
- const groupSelector = options.operation.group ?? 0;
1258
- const scored = [];
1259
- for (let index = 0; index < lines.length; index += 1) {
1260
- const line = lines[index] ?? "";
1261
- let scoreText = line;
1262
- if (options.compiled !== undefined) {
1263
- options.compiled.regex.lastIndex = 0;
1264
- const match = options.compiled.regex.exec(line);
1265
- if (!match) {
1266
- continue;
1267
- }
1268
- if (match[0].length === 0) {
1269
- throw new Error("Regex patterns that produce zero-width matches are not supported.");
1270
- }
1271
- scoreText = matchGroupValue(match, groupSelector, index + 1);
1272
- }
1273
- const score = sort === "numeric" ? Number(scoreText) : scoreText;
1274
- if (sort === "numeric" && (typeof score !== "number" || !Number.isFinite(score))) {
1275
- throw new Error(`topN numeric score on source line ${index + 1} is not finite: ${scoreText}`);
1276
- }
1277
- scored.push({ lineNumber: index + 1, line, score });
1278
- }
1279
- const sorted = [...scored].sort((left, right) => compareTopNEntries(left, right, sort, order));
1280
- const selected = sorted.slice(0, options.operation.limit);
1281
- const parts = [
1282
- "# freeflow_search action=transform topN",
1283
- `source: ${options.sourceLabel}`,
1284
- ...(options.compiled !== undefined
1285
- ? [`pattern: ${formatPattern(options.compiled)}`, `group: ${String(groupSelector)}`]
1286
- : []),
1287
- `sort: ${sort}`,
1288
- `order: ${order}`,
1289
- `limit: ${options.operation.limit}`,
1290
- `matchedLines: ${scored.length}`,
1291
- `returnedLines: ${selected.length}`,
1292
- `truncated: ${selected.length < scored.length}`,
1293
- "",
1294
- ];
1295
- for (const entry of selected) {
1296
- parts.push(`${entry.lineNumber}| score=${String(entry.score)} | ${entry.line}`);
1297
- }
1298
- return {
1299
- text: `${parts.join("\n")}\n`,
1300
- summary: `Transformed topN from vaulted ${sourceStreamLabel(options.sourceLabel)} output: returned ${selected.length} of ${scored.length} matched line(s).`,
1301
- stats: {
1302
- matches: selected.length,
1303
- matchedLines: scored.length,
1304
- matchedLineNumbers: selected.map((entry) => entry.lineNumber),
1305
- truncated: selected.length < scored.length,
1306
- },
1307
- };
1308
- }
1309
- function transformExtractUrls(options) {
1310
- const maxMatches = options.operation.maxMatches ?? DEFAULT_MAX_EXTRACT_MATCHES;
1311
- const dedupe = options.operation.dedupe ?? false;
1312
- const lines = splitLines(options.text);
1313
- const seen = new Set();
1314
- const entries = [];
1315
- let matches = 0;
1316
- let truncated = false;
1317
- lineLoop: for (let index = 0; index < lines.length; index += 1) {
1318
- const line = lines[index] ?? "";
1319
- URL_PATTERN.lastIndex = 0;
1320
- let match;
1321
- while ((match = URL_PATTERN.exec(line)) !== null) {
1322
- const rawUrl = match[0];
1323
- const url = trimUrl(rawUrl);
1324
- if (url.length === 0) {
1325
- continue;
1326
- }
1327
- matches += 1;
1328
- if (!dedupe || !seen.has(url)) {
1329
- seen.add(url);
1330
- entries.push({ lineNumber: index + 1, url });
1331
- }
1332
- if (matches >= maxMatches) {
1333
- truncated = index < lines.length - 1 || URL_PATTERN.lastIndex < line.length;
1334
- break lineLoop;
1335
- }
1336
- }
1337
- }
1338
- const parts = [
1339
- "# freeflow_search action=transform extractUrls",
1340
- `source: ${options.sourceLabel}`,
1341
- `dedupe: ${dedupe}`,
1342
- `maxMatches: ${maxMatches}`,
1343
- `matches: ${matches}`,
1344
- `returnedUrls: ${entries.length}`,
1345
- `truncated: ${truncated}`,
1346
- "",
1347
- ];
1348
- for (const entry of entries) {
1349
- parts.push(`${entry.lineNumber}| ${entry.url}`);
1350
- }
1351
- return {
1352
- text: `${parts.join("\n")}\n`,
1353
- summary: `Transformed extractUrls from vaulted ${sourceStreamLabel(options.sourceLabel)} output: ${entries.length} URL(s) returned from ${matches} match(es).`,
1354
- stats: {
1355
- matches: entries.length,
1356
- matchedLines: new Set(entries.map((entry) => entry.lineNumber)).size,
1357
- matchedLineNumbers: entries.map((entry) => entry.lineNumber),
1358
- truncated,
1359
- },
1360
- };
1361
- }
1362
- function transformExtractCitations(options) {
1363
- const maxMatches = options.operation.maxMatches ?? DEFAULT_MAX_EXTRACT_MATCHES;
1364
- const lines = splitLines(options.text);
1365
- const entries = [];
1366
- let truncated = false;
1367
- lineLoop: for (let index = 0; index < lines.length; index += 1) {
1368
- const line = lines[index] ?? "";
1369
- const lineEntries = citationEntriesForLine(line, index + 1);
1370
- for (let entryIndex = 0; entryIndex < lineEntries.length; entryIndex += 1) {
1371
- const entry = lineEntries[entryIndex];
1372
- if (entry === undefined) {
1373
- continue;
1374
- }
1375
- entries.push(entry);
1376
- if (entries.length >= maxMatches) {
1377
- truncated = entryIndex < lineEntries.length - 1 || index < lines.length - 1;
1378
- break lineLoop;
1379
- }
1380
- }
1381
- }
1382
- const parts = [
1383
- "# freeflow_search action=transform extractCitations",
1384
- `source: ${options.sourceLabel}`,
1385
- `maxMatches: ${maxMatches}`,
1386
- `citations: ${entries.length}`,
1387
- `truncated: ${truncated}`,
1388
- "",
1389
- ];
1390
- for (const entry of entries) {
1391
- parts.push(
1392
- entry.target === undefined
1393
- ? `${entry.lineNumber}| ${entry.type} | ${entry.label}`
1394
- : `${entry.lineNumber}| ${entry.type} | ${entry.label} | ${entry.target}`,
1395
- );
1396
- }
1397
- return {
1398
- text: `${parts.join("\n")}\n`,
1399
- summary: `Transformed extractCitations from vaulted ${sourceStreamLabel(options.sourceLabel)} output: ${entries.length} citation(s) returned.`,
1400
- stats: {
1401
- matches: entries.length,
1402
- matchedLines: new Set(entries.map((entry) => entry.lineNumber)).size,
1403
- matchedLineNumbers: entries.map((entry) => entry.lineNumber),
1404
- truncated,
1405
- },
1406
- };
1407
- }
1408
- function transformLineStats(options) {
1409
- const lines = splitLines(options.text);
1410
- let blankLines = 0;
1411
- let maxLineBytes = 0;
1412
- let maxLineNumber = 0;
1413
- lines.forEach((line, index) => {
1414
- if (line.trim().length === 0) {
1415
- blankLines += 1;
1416
- }
1417
- const lineBytes = byteLength(line);
1418
- if (lineBytes > maxLineBytes) {
1419
- maxLineBytes = lineBytes;
1420
- maxLineNumber = index + 1;
1421
- }
1422
- });
1423
- const nonEmptyLines = lines.length - blankLines;
1424
- const text = [
1425
- "# freeflow_search action=transform lineStats",
1426
- `source: ${options.sourceLabel}`,
1427
- `lines: ${lines.length}`,
1428
- `nonEmptyLines: ${nonEmptyLines}`,
1429
- `blankLines: ${blankLines}`,
1430
- `maxLineBytes: ${maxLineBytes}`,
1431
- `maxLineNumber: ${maxLineNumber}`,
1432
- "",
1433
- ].join("\n");
1434
- return {
1435
- text,
1436
- summary: `Transformed lineStats from vaulted ${sourceStreamLabel(options.sourceLabel)} output: ${lines.length} line(s), ${nonEmptyLines} non-empty, ${blankLines} blank.`,
1437
- stats: {
1438
- matches: lines.length,
1439
- matchedLines: lines.length,
1440
- matchedLineNumbers: [],
1441
- truncated: false,
1442
- },
1443
- };
1444
- }
1445
- function transformSizeStats(options) {
1446
- const bytes = byteLength(options.text);
1447
- const utf16CodeUnits = options.text.length;
1448
- const codePoints = Array.from(options.text).length;
1449
- const lines = countLines(options.text);
1450
- const hash = hashText(options.text);
1451
- const text = [
1452
- "# freeflow_search action=transform sizeStats",
1453
- `source: ${options.sourceLabel}`,
1454
- `bytes: ${bytes}`,
1455
- `utf16CodeUnits: ${utf16CodeUnits}`,
1456
- `codePoints: ${codePoints}`,
1457
- `lines: ${lines}`,
1458
- `sha256: ${hash}`,
1459
- "",
1460
- ].join("\n");
1461
- return {
1462
- text,
1463
- summary: `Transformed sizeStats from vaulted ${sourceStreamLabel(options.sourceLabel)} output: ${bytes} byte(s), ${utf16CodeUnits} code unit(s), ${lines} line(s).`,
1464
- stats: {
1465
- matches: bytes,
1466
- matchedLines: lines,
1467
- matchedLineNumbers: [],
1468
- truncated: false,
1469
- },
1470
- };
1471
- }
1472
- function transformJsonExtract(options) {
1473
- let parsed;
1474
- try {
1475
- parsed = JSON.parse(options.text);
1476
- } catch (error) {
1477
- throw new Error(`Invalid JSON source for jsonExtract: ${errorMessage(error)}`);
1478
- }
1479
- const resolved = resolveJsonSelector(parsed, options.prepared.segments);
1480
- if (!resolved.ok) {
1481
- throw new Error(`JSON selector ${options.prepared.selector} did not resolve: ${resolved.message}`);
1482
- }
1483
- const valueType = jsonValueType(resolved.value);
1484
- const valueText = `${JSON.stringify(resolved.value, null, 2)}\n`;
1485
- const text = [
1486
- "# freeflow_search action=transform jsonExtract",
1487
- `source: ${options.sourceLabel}`,
1488
- `selectorKind: ${options.prepared.selectorKind}`,
1489
- `selector: ${options.prepared.selector}`,
1490
- `valueType: ${valueType}`,
1491
- "",
1492
- valueText,
1493
- ].join("\n");
1494
- return {
1495
- text,
1496
- summary: `Transformed jsonExtract from vaulted ${sourceStreamLabel(options.sourceLabel)} output using ${options.prepared.selectorKind} ${options.prepared.selector}.`,
1497
- stats: {
1498
- matches: 1,
1499
- matchedLines: 1,
1500
- matchedLineNumbers: [],
1501
- truncated: false,
1502
- },
1503
- };
1504
- }
1505
- function collectMatches(lines, regex, maxMatches) {
1506
- let matches = 0;
1507
- let truncated = false;
1508
- const matchedLineNumbers = [];
1509
- lineLoop: for (let index = 0; index < lines.length; index += 1) {
1510
- const line = lines[index] ?? "";
1511
- let lineMatches = 0;
1512
- regex.lastIndex = 0;
1513
- let match;
1514
- while ((match = regex.exec(line)) !== null) {
1515
- if (match[0].length === 0) {
1516
- throw new Error("Regex patterns that produce zero-width matches are not supported.");
1517
- }
1518
- lineMatches += 1;
1519
- matches += 1;
1520
- if (maxMatches !== undefined && matches >= maxMatches) {
1521
- truncated = index < lines.length - 1 || regex.lastIndex < line.length;
1522
- break;
1523
- }
1524
- }
1525
- if (lineMatches > 0) {
1526
- matchedLineNumbers.push(index + 1);
1527
- }
1528
- if (maxMatches !== undefined && matches >= maxMatches) {
1529
- break lineLoop;
1530
- }
1531
- }
1532
- return {
1533
- matches,
1534
- matchedLines: matchedLineNumbers.length,
1535
- matchedLineNumbers,
1536
- truncated,
1537
- };
1538
- }
1539
- function mergeLineWindows(windows) {
1540
- const sorted = windows
1541
- .filter((window) => window.start <= window.end)
1542
- .sort((left, right) => left.start - right.start || left.end - right.end);
1543
- const merged = [];
1544
- for (const window of sorted) {
1545
- const previous = merged[merged.length - 1];
1546
- if (!previous || window.start > previous.end + 1) {
1547
- merged.push({ ...window });
1548
- } else {
1549
- previous.end = Math.max(previous.end, window.end);
1550
- }
1551
- }
1552
- return merged;
1553
- }
1554
- function routeTransformedText(options) {
1555
- const caps =
1556
- options.preserve === "full"
1557
- ? {
1558
- maxLines: Number.MAX_SAFE_INTEGER,
1559
- maxExcerptBytes: options.thresholds.largeOutputBytes,
1560
- maxLineBytes: options.thresholds.largeOutputBytes,
1561
- }
1562
- : {
1563
- maxLines: options.thresholds.largeOutputLines,
1564
- maxExcerptBytes: options.thresholds.largeOutputBytes,
1565
- maxLineBytes: options.thresholds.largeOutputBytes,
1566
- };
1567
- const bounded = assembleTextEvidence({ stream: "combined", text: options.text, caps });
1568
- const outputBytes = byteLength(options.text);
1569
- const outputLines = countLines(options.text);
1570
- const routingStatus = bounded.compressed || bounded.fidelity === "lossy" ? "partial" : "routed";
1571
- const sourceLabel = `${options.source.outputId}:${options.source.stream ?? "combined"}`;
1572
- const evidence = bounded.importantLines.map((line, index) => ({
1573
- id: evidenceId(options.outputId, line.lines, index),
1574
- source: { kind: "vault", outputId: options.outputId, stream: "raw" },
1575
- path: `${options.outputId}:raw`,
1576
- lines: line.lines,
1577
- excerpt: line.excerpt,
1578
- why:
1579
- routingStatus === "partial"
1580
- ? `Bounded transformed ${options.operationKind} output from source ${sourceLabel}; exact transformed content is recoverable from the vault and source lineage is preserved.`
1581
- : `Transformed exact ${options.operationKind} output from source ${sourceLabel} within routing caps; source lineage is preserved.`,
1582
- window: routingStatus === "partial" ? "small" : "exact",
1583
- expandable: true,
1584
- }));
1585
- return {
1586
- routingStatus,
1587
- reason:
1588
- routingStatus === "partial"
1589
- ? `Transformed output from operation ${options.operationKind} over source ${sourceLabel} (${outputBytes} bytes, ${outputLines} lines) was vaulted; bounded evidence was returned with exact recovery.`
1590
- : `Transformed output from operation ${options.operationKind} over source ${sourceLabel} (${outputBytes} bytes, ${outputLines} lines) was vaulted and returned within routing caps.`,
1591
- evidence,
1592
- };
1593
- }
1594
- function citationEntriesForLine(line, lineNumber) {
1595
- const entries = [];
1596
- const inlineLinkPattern = /\[([^\]\n]+)\]\((https?:\/\/[^\s)]+)\)/gi;
1597
- let inlineMatch;
1598
- while ((inlineMatch = inlineLinkPattern.exec(line)) !== null) {
1599
- const label = inlineMatch[1];
1600
- const target = inlineMatch[2];
1601
- if (label !== undefined && target !== undefined) {
1602
- entries.push({ lineNumber, type: "markdown-link", label, target: trimUrl(target) });
1603
- }
1604
- }
1605
- const citeKeyPattern = /\[@([A-Za-z0-9_.:#-]+)\]/g;
1606
- let citeKeyMatch;
1607
- while ((citeKeyMatch = citeKeyPattern.exec(line)) !== null) {
1608
- const label = citeKeyMatch[1];
1609
- if (label !== undefined) {
1610
- entries.push({ lineNumber, type: "citekey", label });
1611
- }
1612
- }
1613
- const footnote = /^\s*\[\^([^\]\n]+)\]:\s*(.+?)\s*$/.exec(line);
1614
- if (footnote?.[1] !== undefined && footnote[2] !== undefined) {
1615
- entries.push({ lineNumber, type: "footnote", label: footnote[1], target: footnote[2] });
1616
- }
1617
- const reference = /^\s*\[([^\]^][^\]\n]*)\]:\s*(.+?)\s*$/.exec(line);
1618
- if (reference?.[1] !== undefined && reference[2] !== undefined) {
1619
- entries.push({ lineNumber, type: "reference", label: reference[1], target: reference[2] });
1620
- }
1621
- return entries;
1622
- }
1623
- function trimUrl(url) {
1624
- return url.replace(/[.,;:!?]+$/g, "");
1625
- }
1626
- function matchGroupValue(match, group, lineNumber) {
1627
- if (typeof group === "number") {
1628
- const value = match[group];
1629
- if (value === undefined) {
1630
- throw new Error(`Regex group ${group} did not resolve on source line ${lineNumber}.`);
1631
- }
1632
- return value;
1633
- }
1634
- const value = match.groups?.[group];
1635
- if (value === undefined) {
1636
- throw new Error(`Regex group ${group} did not resolve on source line ${lineNumber}.`);
1637
- }
1638
- return value;
1639
- }
1640
- function compareTopNEntries(left, right, sort, order) {
1641
- let comparison;
1642
- if (sort === "numeric") {
1643
- comparison = left.score - right.score;
1644
- } else {
1645
- const leftText = String(left.score);
1646
- const rightText = String(right.score);
1647
- comparison = leftText < rightText ? -1 : leftText > rightText ? 1 : 0;
1648
- }
1649
- if (comparison !== 0) {
1650
- return order === "desc" ? -comparison : comparison;
1651
- }
1652
- return left.lineNumber - right.lineNumber;
1653
- }
1654
- function parseJsonPointer(pointer) {
1655
- if (pointer === "") {
1656
- return { ok: true, segments: [] };
1657
- }
1658
- if (!pointer.startsWith("/")) {
1659
- return { ok: false, message: "JSON pointer must be empty or start with /." };
1660
- }
1661
- const segments = [];
1662
- for (const rawSegment of pointer.slice(1).split("/")) {
1663
- if (/~(?![01])/.test(rawSegment)) {
1664
- return { ok: false, message: "JSON pointer contains an invalid escape; use ~0 for ~ and ~1 for /." };
1665
- }
1666
- segments.push({ kind: "property", key: rawSegment.replace(/~1/g, "/").replace(/~0/g, "~") });
1667
- }
1668
- return { ok: true, segments };
1669
- }
1670
- function parseJsonPath(path) {
1671
- if (path.length === 0 || path[0] !== "$") {
1672
- return { ok: false, message: "JSON path must start with $." };
1673
- }
1674
- const segments = [];
1675
- let index = 1;
1676
- while (index < path.length) {
1677
- const char = path[index];
1678
- if (char === ".") {
1679
- const parsed = parseJsonPathProperty(path, index + 1);
1680
- if (!parsed.ok) {
1681
- return parsed;
1682
- }
1683
- segments.push({ kind: "property", key: parsed.key });
1684
- index = parsed.nextIndex;
1685
- continue;
1686
- }
1687
- if (char === "[") {
1688
- const parsed = parseJsonPathBracket(path, index + 1);
1689
- if (!parsed.ok) {
1690
- return parsed;
1691
- }
1692
- segments.push(parsed.segment);
1693
- index = parsed.nextIndex;
1694
- continue;
1695
- }
1696
- return { ok: false, message: `Unexpected token ${char} at offset ${index}.` };
1697
- }
1698
- return { ok: true, segments };
1699
- }
1700
- function parseJsonPathProperty(path, startIndex) {
1701
- const match = /^[A-Za-z_$][A-Za-z0-9_$-]*/.exec(path.slice(startIndex));
1702
- if (!match?.[0]) {
1703
- return { ok: false, message: `Expected property name after . at offset ${startIndex - 1}.` };
1704
- }
1705
- return { ok: true, key: match[0], nextIndex: startIndex + match[0].length };
1706
- }
1707
- function parseJsonPathBracket(path, startIndex) {
1708
- const first = path[startIndex];
1709
- if (first === '"') {
1710
- return parseJsonPathQuotedProperty(path, startIndex);
1711
- }
1712
- if (first === "'") {
1713
- return { ok: false, message: `Use double-quoted bracket properties at offset ${startIndex}.` };
1714
- }
1715
- const closeIndex = path.indexOf("]", startIndex);
1716
- if (closeIndex === -1) {
1717
- return { ok: false, message: `Expected ] for bracket selector at offset ${startIndex - 1}.` };
1718
- }
1719
- const indexText = path.slice(startIndex, closeIndex);
1720
- if (!/^(0|[1-9][0-9]*)$/.test(indexText)) {
1721
- return { ok: false, message: `Expected non-negative array index in bracket selector at offset ${startIndex - 1}.` };
1722
- }
1723
- return { ok: true, segment: { kind: "index", index: Number(indexText) }, nextIndex: closeIndex + 1 };
1724
- }
1725
- function parseJsonPathQuotedProperty(path, quoteIndex) {
1726
- let escaped = false;
1727
- for (let index = quoteIndex + 1; index < path.length; index += 1) {
1728
- const char = path[index];
1729
- if (escaped) {
1730
- escaped = false;
1731
- continue;
1732
- }
1733
- if (char === "\\") {
1734
- escaped = true;
1735
- continue;
1736
- }
1737
- if (char === '"') {
1738
- if (path[index + 1] !== "]") {
1739
- return { ok: false, message: `Expected ] after quoted property at offset ${index}.` };
1740
- }
1741
- const literal = path.slice(quoteIndex, index + 1);
1742
- try {
1743
- const key = JSON.parse(literal);
1744
- if (typeof key !== "string") {
1745
- return { ok: false, message: "Expected quoted JSON path property to decode to a string." };
1746
- }
1747
- return { ok: true, segment: { kind: "property", key }, nextIndex: index + 2 };
1748
- } catch (error) {
1749
- return { ok: false, message: `Invalid quoted property string: ${errorMessage(error)}` };
1750
- }
1751
- }
1752
- }
1753
- return { ok: false, message: `Unterminated quoted property at offset ${quoteIndex}.` };
1754
- }
1755
- function resolveJsonSelector(value, segments) {
1756
- let current = value;
1757
- for (const segment of segments) {
1758
- if (segment.kind === "index") {
1759
- if (!Array.isArray(current)) {
1760
- return { ok: false, message: `array index ${segment.index} cannot be applied to ${jsonValueType(current)}.` };
1761
- }
1762
- if (segment.index >= current.length) {
1763
- return { ok: false, message: `array index ${segment.index} is outside length ${current.length}.` };
1764
- }
1765
- current = current[segment.index];
1766
- continue;
1767
- }
1768
- if (Array.isArray(current)) {
1769
- if (!isArrayIndexSegment(segment.key)) {
1770
- return { ok: false, message: `array source requires numeric pointer segment, got ${segment.key}.` };
1771
- }
1772
- const arrayIndex = Number(segment.key);
1773
- if (arrayIndex >= current.length) {
1774
- return { ok: false, message: `array index ${arrayIndex} is outside length ${current.length}.` };
1775
- }
1776
- current = current[arrayIndex];
1777
- continue;
1778
- }
1779
- if (!isJsonObject(current)) {
1780
- return { ok: false, message: `property ${segment.key} cannot be applied to ${jsonValueType(current)}.` };
1781
- }
1782
- if (!Object.prototype.hasOwnProperty.call(current, segment.key)) {
1783
- return { ok: false, message: `property ${segment.key} is not present.` };
1784
- }
1785
- current = current[segment.key];
1786
- }
1787
- return { ok: true, value: current };
1788
- }
1789
- function isJsonObject(value) {
1790
- return typeof value === "object" && value !== null && !Array.isArray(value);
1791
- }
1792
- function isArrayIndexSegment(value) {
1793
- return /^(0|[1-9][0-9]*)$/.test(value);
1794
- }
1795
- function jsonValueType(value) {
1796
- if (value === null) {
1797
- return "null";
1798
- }
1799
- if (Array.isArray(value)) {
1800
- return "array";
1801
- }
1802
- return typeof value;
1803
- }
1804
- function effectiveScriptTransformConfig(config) {
1805
- if (config === undefined) {
1806
- return {
1807
- ...DEFAULT_SCRIPT_TRANSFORM_CONFIG,
1808
- languages: [...DEFAULT_SCRIPT_TRANSFORM_CONFIG.languages],
1809
- limits: { ...DEFAULT_SCRIPT_TRANSFORM_CONFIG.limits },
1810
- };
1811
- }
1812
- return {
1813
- ...config,
1814
- languages: [...config.languages],
1815
- limits: { ...config.limits },
1816
- };
1817
- }
1818
- function effectiveScriptLimits(config, input) {
1819
- return {
1820
- timeoutMs: boundedScriptLimit(input?.timeoutMs, config.limits.timeoutMs, MAX_SCRIPT_TRANSFORM_LIMITS.timeoutMs),
1821
- maxInputBytes: boundedScriptLimit(
1822
- input?.maxInputBytes,
1823
- config.limits.maxInputBytes,
1824
- MAX_SCRIPT_TRANSFORM_LIMITS.maxInputBytes,
1825
- ),
1826
- maxOutputBytes: boundedScriptLimit(
1827
- input?.maxOutputBytes,
1828
- config.limits.maxOutputBytes,
1829
- MAX_SCRIPT_TRANSFORM_LIMITS.maxOutputBytes,
1830
- ),
1831
- };
1832
- }
1833
- function boundedScriptLimit(value, configured, max) {
1834
- const configuredLimit = Math.min(configured, max);
1835
- if (!Number.isInteger(value) || Number(value) <= 0) {
1836
- return configuredLimit;
1837
- }
1838
- return Math.min(Number(value), configuredLimit);
1839
- }
1840
- function resolveSourceStream(record, requested) {
1841
- if (record.kind === "command") {
1842
- const stream = requested ?? "combined";
1843
- if (stream === "raw") {
1844
- return { ok: false, message: "Command vault sources support stdout, stderr, or combined streams, not raw." };
1845
- }
1846
- return { ok: true, stream };
1847
- }
1848
- if (record.kind === "text") {
1849
- const stream = requested ?? "raw";
1850
- if (stream !== "raw") {
1851
- return { ok: false, message: "Text vault sources support only the raw stream." };
1852
- }
1853
- return { ok: true, stream };
1854
- }
1855
- return {
1856
- ok: false,
1857
- message: "Repo file reference vault records store metadata only and cannot be used as transform text sources.",
1858
- };
1859
- }
1860
- function lineageForSource(record, operation) {
1861
- return {
1862
- sourceRecordIds: [record.recordId],
1863
- sourceOutputIds: [record.outputId],
1864
- operation: operation.kind === "script" ? `script:${operation.language}` : operation.kind,
1865
- operationHash: operationHash(operation),
1866
- };
1867
- }
1868
- function lineageForScriptInput(input, limits) {
1869
- return {
1870
- sourceOutputIds: input.sources.map((source) => source.outputId),
1871
- operation: `script:${input.operation.language}`,
1872
- operationHash: operationHashForScript(input.operation, input.sources, limits ?? input.limits),
1873
- };
1874
- }
1875
- function lineageForResolvedScriptSources(sources, input, limits) {
1876
- return {
1877
- sourceRecordIds: sources.map((source) => source.recordId),
1878
- sourceOutputIds: sources.map((source) => source.outputId),
1879
- operation: `script:${input.operation.language}`,
1880
- operationHash: operationHashForScript(input.operation, input.sources, limits, sources),
1881
- };
1882
- }
1883
- function lineageFromInput(input) {
1884
- if (!isRecord(input) || !isRecord(input.operation)) {
1885
- return undefined;
1886
- }
1887
- if (input.operation.kind === "script" && Array.isArray(input.sources)) {
1888
- const sources = input.sources.filter(isRecord);
1889
- const outputIds = sources.map((source) => source.outputId).filter((outputId) => typeof outputId === "string");
1890
- return {
1891
- ...(outputIds.length > 0 ? { sourceOutputIds: outputIds } : {}),
1892
- operation: typeof input.operation.language === "string" ? `script:${input.operation.language}` : "script",
1893
- };
1894
- }
1895
- if (!isRecord(input.source)) {
1896
- return undefined;
1897
- }
1898
- const lineage = {};
1899
- if (typeof input.source.outputId === "string") {
1900
- lineage.sourceOutputIds = [input.source.outputId];
1901
- }
1902
- if (typeof input.operation.kind === "string") {
1903
- lineage.operation = input.operation.kind;
1904
- }
1905
- return Object.keys(lineage).length > 0 ? lineage : undefined;
1906
- }
1907
- function operationSummary(operation) {
1908
- if (operation.kind === "script") {
1909
- return scriptOperationSummary(operation);
1910
- }
1911
- if (operation.kind === "regexFilter") {
1912
- return {
1913
- kind: operation.kind,
1914
- pattern: operation.pattern,
1915
- ...(operation.flags !== undefined ? { flags: normalizeRegexFlags(operation.flags) } : {}),
1916
- contextLines: operation.contextLines ?? DEFAULT_CONTEXT_LINES,
1917
- maxMatches: operation.maxMatches ?? DEFAULT_MAX_MATCHES,
1918
- };
1919
- }
1920
- if (operation.kind === "countMatches") {
1921
- return {
1922
- kind: operation.kind,
1923
- pattern: operation.pattern,
1924
- ...(operation.flags !== undefined ? { flags: normalizeRegexFlags(operation.flags) } : {}),
1925
- };
1926
- }
1927
- if (operation.kind === "groupByRegex") {
1928
- return {
1929
- kind: operation.kind,
1930
- pattern: operation.pattern,
1931
- ...(operation.flags !== undefined ? { flags: normalizeRegexFlags(operation.flags) } : {}),
1932
- group: operation.group ?? 1,
1933
- maxGroups: operation.maxGroups ?? DEFAULT_MAX_GROUPS,
1934
- maxLinesPerGroup: operation.maxLinesPerGroup ?? DEFAULT_MAX_LINES_PER_GROUP,
1935
- };
1936
- }
1937
- if (operation.kind === "dedupe") {
1938
- return {
1939
- kind: operation.kind,
1940
- trim: operation.trim ?? false,
1941
- caseSensitive: operation.caseSensitive ?? true,
1942
- maxLines: operation.maxLines ?? DEFAULT_MAX_DEDUPE_LINES,
1943
- };
1944
- }
1945
- if (operation.kind === "topN") {
1946
- return {
1947
- kind: operation.kind,
1948
- limit: operation.limit,
1949
- ...(operation.pattern !== undefined ? { pattern: operation.pattern } : {}),
1950
- ...(operation.flags !== undefined ? { flags: normalizeRegexFlags(operation.flags) } : {}),
1951
- ...(operation.group !== undefined ? { group: operation.group } : {}),
1952
- sort: operation.sort ?? "text",
1953
- order: operation.order ?? "asc",
1954
- };
1955
- }
1956
- if (operation.kind === "extractUrls") {
1957
- return {
1958
- kind: operation.kind,
1959
- dedupe: operation.dedupe ?? false,
1960
- maxMatches: operation.maxMatches ?? DEFAULT_MAX_EXTRACT_MATCHES,
1961
- };
1962
- }
1963
- if (operation.kind === "extractCitations") {
1964
- return {
1965
- kind: operation.kind,
1966
- maxMatches: operation.maxMatches ?? DEFAULT_MAX_EXTRACT_MATCHES,
1967
- };
1968
- }
1969
- if (operation.kind === "lineStats" || operation.kind === "sizeStats") {
1970
- return { kind: operation.kind };
1971
- }
1972
- return {
1973
- kind: operation.kind,
1974
- ...(operation.pointer !== undefined ? { pointer: operation.pointer } : {}),
1975
- ...(operation.path !== undefined ? { path: operation.path } : {}),
1976
- };
1977
- }
1978
- function operationHash(operation) {
1979
- return `sha256_${hash(JSON.stringify(operationSummary(operation)))}`;
1980
- }
1981
- function operationHashForScript(operation, sources, limits, resolvedSources = []) {
1982
- return `sha256_${hash(
1983
- JSON.stringify({
1984
- schemaVersion: 1,
1985
- operation: scriptOperationSummary(operation),
1986
- sources: sources.map((source) => ({ alias: source.alias, outputId: source.outputId, stream: source.stream })),
1987
- resolvedSources: resolvedSources.map((source) => ({
1988
- alias: source.alias,
1989
- outputId: source.outputId,
1990
- recordId: source.recordId,
1991
- stream: source.stream,
1992
- bytes: source.bytes,
1993
- contentHashSha256: source.contentHashSha256,
1994
- textSha256: source.textSha256,
1995
- })),
1996
- limits,
1997
- }),
1998
- )}`;
1999
- }
2000
- function scriptOperationSummary(operation) {
2001
- return {
2002
- kind: "script",
2003
- language: operation.language,
2004
- codeSha256: `sha256_${hash(operation.code)}`,
2005
- ...(operation.label !== undefined ? { label: operation.label } : {}),
2006
- };
2007
- }
2008
- function normalizeRegexFlags(flags) {
2009
- return [...new Set((flags ?? "").replace(/g/g, "").split(""))].sort().join("");
2010
- }
2011
- function ensureGlobalFlag(flags) {
2012
- return flags.includes("g") ? flags : `${flags}g`;
2013
- }
2014
- function formatPattern(compiled) {
2015
- return `/${compiled.displayPattern}/${compiled.flags}`;
2016
- }
2017
- function sourceStreamLabel(sourceLabel) {
2018
- const [, stream] = sourceLabel.split(":");
2019
- return stream ?? "combined";
2020
- }
2021
- function transformValidationFailureWithOptionalLineage(options) {
2022
- const failureOptions = {
2023
- message: options.message,
2024
- preserve: options.preserve,
2025
- decisionSeed: options.decisionSeed,
2026
- };
2027
- if (options.lineage !== undefined) {
2028
- return transformValidationFailure({ ...failureOptions, lineage: options.lineage });
2029
- }
2030
- return transformValidationFailure(failureOptions);
2031
- }
2032
- function transformSourceUnavailableFailureWithOptionalLineage(options) {
2033
- const failureOptions = {
2034
- message: options.message,
2035
- preserve: options.preserve,
2036
- decisionSeed: options.decisionSeed,
2037
- };
2038
- if (options.lineage !== undefined) {
2039
- return transformSourceUnavailableFailure({ ...failureOptions, lineage: options.lineage });
2040
- }
2041
- return transformSourceUnavailableFailure(failureOptions);
2042
- }
2043
- function validationMessage(issues) {
2044
- return `Invalid transform input: ${issues.map((issue) => `${issue.path} ${issue.message}`).join("; ")}`;
2045
- }
2046
- function isScriptTransformInput(input) {
2047
- return input.operation.kind === "script";
2048
- }
2049
- function isOutputStream(value) {
2050
- return value === "stdout" || value === "stderr" || value === "combined" || value === "raw";
2051
- }
2052
- function isStringIn(value, allowed) {
2053
- return typeof value === "string" && allowed.includes(value);
2054
- }
2055
- function isRecord(value) {
2056
- return typeof value === "object" && value !== null && !Array.isArray(value);
2057
- }
2058
- function evidenceId(outputId, lines, index) {
2059
- return `ev_${hash(`${outputId}:${lines}:${index}`).slice(0, 16)}`;
2060
- }
2061
- function decisionId(...parts) {
2062
- return `ffdec_${hash(parts.join("\0")).slice(0, 16)}`;
2063
- }
2064
- function hash(value) {
2065
- return createHash("sha256").update(value).digest("hex");
2066
- }
2067
- function hashText(value) {
2068
- return createHash("sha256").update(value).digest("hex");
2069
- }
2070
- function errorMessage(error) {
2071
- return error instanceof Error ? error.message : String(error);
2072
- }