@hassangameryt/freeflow 0.3.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +4 -4
- package/.claude-plugin/plugin.json +6 -2
- package/.codex-plugin/plugin.json +9 -9
- package/README.md +190 -127
- package/assets/feedback-loop.png +0 -0
- package/assets/freeflow_workflow_map.png +0 -0
- package/assets/workflow.png +0 -0
- package/{skills → capabilities}/output-router/SKILL.md +21 -17
- package/{skills → capabilities}/output-router/references/safety-policy.md +1 -1
- package/capabilities/output-router/references/setup.md +110 -0
- package/command-surface.json +62 -7
- package/hooks/freeflow-runtime-context.mjs +572 -154
- package/hooks/hooks.json +26 -1
- package/hooks/tests/check-runtime-context-hook.sh +375 -0
- package/package.json +24 -24
- package/pi-extension/dist/index.js +264 -80
- package/pi-extension/dist/integrations/mcp-config.js +29 -0
- package/pi-extension/dist/output-router/host-producer-identification.js +232 -0
- package/pi-extension/dist/output-router/native-safety-net.js +216 -0
- package/pi-extension/dist/output-router/observed-tool-routing.js +122 -0
- package/pi-extension/dist/output-router/renderers.js +1023 -0
- package/pi-extension/dist/output-router/router-tools.js +554 -0
- package/pi-extension/dist/output-router/schemas.js +383 -0
- package/pi-extension/dist/output-router/status.js +744 -0
- package/pi-extension/dist/output-router/utils.js +502 -0
- package/pi-extension/dist/runtime/runtime-context.js +798 -0
- package/pi-extension/dist/settings/settings-tui.js +314 -0
- package/pi-extension/dist/settings/settings-ui.js +1487 -0
- package/pi-extension/freeflow/index.js +1 -0
- package/pi-extension/src/index.ts +230 -56
- package/pi-extension/src/{host-producer-identification.ts → output-router/host-producer-identification.ts} +37 -7
- package/pi-extension/src/{native-safety-net.ts → output-router/native-safety-net.ts} +8 -12
- package/pi-extension/src/{observed-tool-routing.ts → output-router/observed-tool-routing.ts} +19 -11
- package/pi-extension/src/{renderers.ts → output-router/renderers.ts} +246 -55
- package/pi-extension/src/{router-tools.ts → output-router/router-tools.ts} +130 -29
- package/pi-extension/src/{schemas.ts → output-router/schemas.ts} +171 -43
- package/pi-extension/src/output-router/status.ts +791 -0
- package/pi-extension/src/{utils.ts → output-router/utils.ts} +80 -18
- package/pi-extension/src/runtime/runtime-context.ts +895 -0
- package/pi-extension/src/settings/settings-tui.ts +408 -0
- package/pi-extension/src/settings/settings-ui.ts +1730 -0
- package/pi-extension/tests/integration/pi-extension.test.js +4020 -0
- package/pi-extension/tests/output-router/observed-producers.test.js +132 -0
- package/pi-extension/tests/output-router/search.test.js +722 -0
- package/pi-extension/tests/settings/settings-tui.test.js +243 -0
- package/router/dist/benchmarks/benchmark-harness.d.ts +16 -13
- package/router/dist/benchmarks/benchmark-harness.js +62 -62
- package/router/dist/benchmarks/benchmarks.d.ts +80 -73
- package/router/dist/benchmarks/benchmarks.js +860 -761
- package/router/dist/benchmarks/codex-qa-benchmarks.d.ts +71 -67
- package/router/dist/benchmarks/codex-qa-benchmarks.js +429 -390
- package/router/dist/benchmarks/command-benchmarks.d.ts +107 -92
- package/router/dist/benchmarks/command-benchmarks.js +616 -571
- package/router/dist/benchmarks/context-mode-normalized-benchmarks.d.ts +64 -56
- package/router/dist/benchmarks/context-mode-normalized-benchmarks.js +679 -613
- package/router/dist/benchmarks/context-mode-real-deep-benchmark.d.ts +103 -95
- package/router/dist/benchmarks/context-mode-real-deep-benchmark.js +1188 -1103
- package/router/dist/benchmarks/context-saving-baseline.d.ts +47 -41
- package/router/dist/benchmarks/context-saving-baseline.js +405 -374
- package/router/dist/benchmarks/index-benchmarks.d.ts +74 -69
- package/router/dist/benchmarks/index-benchmarks.js +761 -661
- package/router/dist/benchmarks/storage-policy-benchmarks.d.ts +80 -69
- package/router/dist/benchmarks/storage-policy-benchmarks.js +549 -469
- package/router/dist/benchmarks/vault-index-benchmarks.d.ts +43 -34
- package/router/dist/benchmarks/vault-index-benchmarks.js +294 -229
- package/router/dist/config/config.d.ts +62 -55
- package/router/dist/config/config.js +552 -373
- package/router/dist/config/router-contract.d.ts +2 -2
- package/router/dist/config/router-contract.js +38 -38
- package/router/dist/config/schema.d.ts +19 -10
- package/router/dist/config/schema.js +586 -538
- package/router/dist/config/types.d.ts +413 -361
- package/router/dist/config/types.js +19 -8
- package/router/dist/evidence/bounded-evidence.d.ts +26 -26
- package/router/dist/evidence/bounded-evidence.js +265 -252
- package/router/dist/evidence/evidence-range-selector.d.ts +16 -14
- package/router/dist/evidence/evidence-range-selector.js +192 -191
- package/router/dist/evidence/evidence-search.d.ts +18 -16
- package/router/dist/evidence/evidence-search.js +401 -377
- package/router/dist/evidence/evidence.d.ts +17 -17
- package/router/dist/evidence/evidence.js +163 -159
- package/router/dist/evidence/failure-contracts.d.ts +25 -15
- package/router/dist/evidence/failure-contracts.js +132 -126
- package/router/dist/evidence/line-ranges.d.ts +17 -15
- package/router/dist/evidence/line-ranges.js +18 -18
- package/router/dist/experiments/local-index.d.ts +47 -41
- package/router/dist/experiments/local-index.js +477 -458
- package/router/dist/local/local-traversal.d.ts +18 -0
- package/router/dist/local/local-traversal.js +334 -0
- package/router/dist/processing/engine.d.ts +94 -70
- package/router/dist/processing/engine.js +462 -364
- package/router/dist/processing/reducers.d.ts +208 -194
- package/router/dist/processing/reducers.js +1434 -1256
- package/router/dist/processing/renderers.d.ts +24 -18
- package/router/dist/processing/renderers.js +161 -152
- package/router/dist/processing/scripts.d.ts +80 -67
- package/router/dist/processing/scripts.js +343 -290
- package/router/dist/repo/repo-traversal.d.ts +9 -9
- package/router/dist/repo/repo-traversal.js +205 -202
- package/router/dist/routing/observed-reducers.d.ts +25 -19
- package/router/dist/routing/observed-reducers.js +559 -545
- package/router/dist/routing/observed-routing.d.ts +23 -14
- package/router/dist/routing/observed-routing.js +319 -318
- package/router/dist/routing/parsers.d.ts +10 -10
- package/router/dist/routing/parsers.js +454 -406
- package/router/dist/routing/run-filters.d.ts +45 -41
- package/router/dist/routing/run-filters.js +215 -207
- package/router/dist/routing/run-reducers.d.ts +45 -26
- package/router/dist/routing/run-reducers.js +102 -90
- package/router/dist/sandbox/adapter-roots.d.ts +34 -26
- package/router/dist/sandbox/adapter-roots.js +43 -41
- package/router/dist/sandbox/eryx-python-adapter.d.ts +9 -5
- package/router/dist/sandbox/eryx-python-adapter.js +659 -588
- package/router/dist/sandbox/jq-wasm-adapter.d.ts +3 -3
- package/router/dist/sandbox/jq-wasm-adapter.js +319 -311
- package/router/dist/sandbox/quickjs-wasi-adapter.d.ts +9 -5
- package/router/dist/sandbox/quickjs-wasi-adapter.js +336 -333
- package/router/dist/sandbox/script-sandbox.d.ts +105 -84
- package/router/dist/sandbox/script-sandbox.js +294 -260
- package/router/dist/setup/script-transform-adapters.d.ts +23 -19
- package/router/dist/setup/script-transform-adapters.js +223 -204
- package/router/dist/tools/batch.d.ts +27 -16
- package/router/dist/tools/batch.js +493 -416
- package/router/dist/tools/run.d.ts +47 -35
- package/router/dist/tools/run.js +1172 -1087
- package/router/dist/tools/search.d.ts +53 -31
- package/router/dist/tools/search.js +1224 -1065
- package/router/dist/transform/engine.d.ts +121 -96
- package/router/dist/transform/engine.js +1853 -1736
- package/router/dist/vault/vault-index.d.ts +97 -78
- package/router/dist/vault/vault-index.js +347 -342
- package/router/dist/vault/vault.d.ts +109 -74
- package/router/dist/vault/vault.js +528 -533
- package/runtime/interaction-contract.md +16 -0
- package/skills/bypass/SKILL.md +52 -38
- package/skills/commit-work/SKILL.md +54 -46
- package/skills/commit-work/references/staging-decisions.md +46 -24
- package/skills/decision-gate/SKILL.md +48 -0
- package/skills/design-for-depth/SKILL.md +75 -59
- package/skills/design-for-depth/references/design-pressure-signals.md +56 -7
- package/skills/design-for-depth/references/interface-design-loop.md +156 -0
- package/skills/design-for-depth/references/software-design-philosophy.md +32 -3
- package/skills/diagnose-failure/SKILL.md +65 -60
- package/skills/diagnose-failure/references/diagnostic-loop-catalog.md +36 -0
- package/skills/diagnose-failure/references/flaky-and-performance.md +30 -18
- package/skills/discuss/SKILL.md +121 -0
- package/skills/discuss/references/checkpoints.md +66 -0
- package/skills/evaluate-skill/SKILL.md +62 -53
- package/skills/evaluate-skill/references/definition-schema.md +345 -0
- package/skills/evaluate-skill/references/evaluation-design.md +86 -0
- package/skills/evaluate-skill/references/execution-and-evidence.md +85 -0
- package/skills/evaluate-skill/references/review-and-revision.md +51 -0
- package/skills/evaluate-skill/scripts/lib/definitions.mjs +296 -0
- package/skills/evaluate-skill/scripts/lib/evidence.mjs +46 -0
- package/skills/evaluate-skill/scripts/lib/grade-comparison.mjs +60 -0
- package/skills/evaluate-skill/scripts/lib/grade-interaction.mjs +148 -0
- package/skills/evaluate-skill/scripts/lib/grade-support.mjs +126 -0
- package/skills/evaluate-skill/scripts/lib/grade-workspace.mjs +192 -0
- package/skills/evaluate-skill/scripts/lib/grade.mjs +56 -0
- package/skills/evaluate-skill/scripts/lib/pi.mjs +489 -0
- package/skills/evaluate-skill/scripts/lib/process.mjs +79 -0
- package/skills/evaluate-skill/scripts/lib/rpc.mjs +444 -0
- package/skills/evaluate-skill/scripts/lib/runner.mjs +346 -0
- package/skills/evaluate-skill/scripts/lib/sandbox.mjs +409 -0
- package/skills/evaluate-skill/scripts/lib/view.mjs +337 -0
- package/skills/evaluate-skill/scripts/pi-guard.mjs +164 -0
- package/skills/evaluate-skill/scripts/skill-eval.mjs +101 -0
- package/skills/execute-work/SKILL.md +99 -0
- package/skills/execute-work/references/code-practices.md +78 -0
- package/skills/execute-work/references/execution-loop.md +77 -0
- package/skills/finish-branch/SKILL.md +117 -0
- package/skills/finish-branch/references/integration-options.md +80 -0
- package/skills/handoff/SKILL.md +65 -39
- package/skills/handoff/references/templates.md +104 -27
- package/skills/launch-work/SKILL.md +116 -0
- package/skills/launch-work/references/launch-readiness.md +85 -0
- package/skills/migration-work/SKILL.md +115 -0
- package/skills/migration-work/references/migration-lifecycle.md +82 -0
- package/skills/mode-contract/SKILL.md +72 -52
- package/skills/release-work/SKILL.md +118 -0
- package/skills/release-work/references/release-evidence.md +78 -0
- package/skills/review-artifact/SKILL.md +77 -69
- package/skills/review-artifact/references/reviewer-prompt.md +133 -42
- package/skills/review-work/SKILL.md +82 -57
- package/skills/review-work/references/reviewer-prompt.md +97 -60
- package/skills/review-work/references/security-risk-lens.md +67 -0
- package/skills/setup-freeflow/SKILL.md +54 -68
- package/skills/setup-freeflow/references/activation-contract.md +84 -69
- package/skills/setup-freeflow/references/host-setup.md +50 -17
- package/skills/simplify-code/SKILL.md +80 -0
- package/skills/simplify-code/references/simplification-patterns.md +70 -0
- package/skills/tdd/SKILL.md +96 -0
- package/skills/tdd/references/test-design.md +96 -0
- package/skills/track-work/SKILL.md +84 -0
- package/skills/track-work/references/working-record-schema.md +378 -0
- package/skills/verify-work/SKILL.md +59 -38
- package/skills/verify-work/references/browser-runtime-evidence.md +55 -0
- package/skills/verify-work/references/integration-evidence.md +51 -0
- package/skills/verify-work/references/performance-evidence.md +57 -0
- package/skills/workflow/SKILL.md +53 -59
- package/skills/workflow/references/domain-skill-composition.md +57 -0
- package/skills/workflow/references/workflow-loop.md +92 -0
- package/skills/write-plan/SKILL.md +85 -58
- package/skills/write-plan/references/plan-shapes.md +132 -46
- package/skills/write-skill/SKILL.md +82 -51
- package/skills/write-skill/references/activation-boundaries.md +62 -0
- package/skills/write-skill/references/agent-first-instructions.md +87 -0
- package/skills/write-skill/references/progressive-disclosure.md +63 -0
- package/skills/write-skill/scripts/lib/skill-author.mjs +653 -0
- package/skills/write-skill/scripts/skill-author.mjs +104 -0
- package/skills/write-spec/SKILL.md +65 -54
- package/skills/write-spec/references/artifact-standards.md +60 -41
- package/skills/write-spec/references/decision-records.md +77 -0
- package/skills/write-spec/references/spec-shapes.md +119 -60
- package/evals/README.md +0 -32
- package/evals/fixtures/copy-conventions.md +0 -31
- package/evals/fixtures/output-router/README.md +0 -12
- package/evals/fixtures/output-router/failed-command-output.txt +0 -9
- package/evals/fixtures/output-router/large-router-manual.md +0 -63
- package/evals/fixtures/output-router/native-large-output.txt +0 -40
- package/evals/fixtures/output-router/noisy-test-output.txt +0 -30
- package/evals/fixtures/output-router/verification-output.txt +0 -8
- package/evals/fixtures/tiny-artifact-review-app/README.md +0 -5
- package/evals/fixtures/tiny-artifact-review-app/docs/billing-policy.md +0 -10
- package/evals/fixtures/tiny-artifact-review-app/docs/specs/billing-webhook-api-draft.md +0 -21
- package/evals/fixtures/tiny-artifact-review-app/docs/specs/failed-payment-grace-period.md +0 -24
- package/evals/fixtures/tiny-artifact-review-app/docs/specs/immediate-downgrade.md +0 -17
- package/evals/fixtures/tiny-artifact-review-app/tests/billing.test.md +0 -8
- package/evals/fixtures/tiny-commit-ambiguous-app/.freeflow-eval-setup.sh +0 -15
- package/evals/fixtures/tiny-commit-ambiguous-app/README.md +0 -3
- package/evals/fixtures/tiny-commit-ambiguous-app/src/billing.ts +0 -5
- package/evals/fixtures/tiny-commit-ambiguous-app/src/settings-page.ts +0 -1
- package/evals/fixtures/tiny-commit-ambiguous-app/tests/settings.test.md +0 -5
- package/evals/fixtures/tiny-commit-app/.freeflow-eval-setup.sh +0 -14
- package/evals/fixtures/tiny-commit-app/README.md +0 -3
- package/evals/fixtures/tiny-commit-app/docs/specs/settings-title.md +0 -5
- package/evals/fixtures/tiny-commit-app/src/billing.ts +0 -5
- package/evals/fixtures/tiny-commit-app/src/settings-page.ts +0 -1
- package/evals/fixtures/tiny-commit-app/tests/settings.test.md +0 -5
- package/evals/fixtures/tiny-commit-everything-app/.freeflow-eval-setup.sh +0 -15
- package/evals/fixtures/tiny-commit-everything-app/README.md +0 -3
- package/evals/fixtures/tiny-commit-everything-app/docs/specs/settings-title.md +0 -5
- package/evals/fixtures/tiny-commit-everything-app/src/billing.ts +0 -5
- package/evals/fixtures/tiny-commit-everything-app/src/settings-page.ts +0 -1
- package/evals/fixtures/tiny-commit-everything-app/tests/settings.test.md +0 -5
- package/evals/fixtures/tiny-commit-staged-app/.freeflow-eval-setup.sh +0 -15
- package/evals/fixtures/tiny-commit-staged-app/README.md +0 -3
- package/evals/fixtures/tiny-commit-staged-app/docs/specs/settings-title.md +0 -5
- package/evals/fixtures/tiny-commit-staged-app/src/billing.ts +0 -5
- package/evals/fixtures/tiny-commit-staged-app/src/settings-page.ts +0 -1
- package/evals/fixtures/tiny-commit-staged-app/tests/settings.test.md +0 -5
- package/evals/fixtures/tiny-commit-staged-mixed-app/.freeflow-eval-setup.sh +0 -15
- package/evals/fixtures/tiny-commit-staged-mixed-app/README.md +0 -3
- package/evals/fixtures/tiny-commit-staged-mixed-app/docs/billing-policy.md +0 -3
- package/evals/fixtures/tiny-commit-staged-mixed-app/docs/release-notes.md +0 -3
- package/evals/fixtures/tiny-commit-staged-mixed-app/src/billing.ts +0 -5
- package/evals/fixtures/tiny-dashboard-stale-app/README.md +0 -11
- package/evals/fixtures/tiny-dashboard-stale-app/src/api-client.ts +0 -13
- package/evals/fixtures/tiny-dashboard-stale-app/src/dashboard-data.ts +0 -19
- package/evals/fixtures/tiny-dashboard-stale-app/tests/dashboard-data.test.md +0 -9
- package/evals/fixtures/tiny-decision-destination-app/README.md +0 -6
- package/evals/fixtures/tiny-decision-destination-app/src/risk.ts +0 -8
- package/evals/fixtures/tiny-decision-destination-app/tests/risk.test.md +0 -4
- package/evals/fixtures/tiny-design-depth-app/README.md +0 -15
- package/evals/fixtures/tiny-design-depth-app/src/routes.ts +0 -25
- package/evals/fixtures/tiny-design-depth-app/src/sendEmail.ts +0 -3
- package/evals/fixtures/tiny-design-depth-app/src/sendSms.ts +0 -3
- package/evals/fixtures/tiny-design-depth-app/src/telemetry.ts +0 -3
- package/evals/fixtures/tiny-design-depth-app/tests/routes.test.md +0 -12
- package/evals/fixtures/tiny-eval-method-app/README.md +0 -8
- package/evals/fixtures/tiny-eval-method-app/evals/reports/review-pr-eval-result.md +0 -5
- package/evals/fixtures/tiny-eval-method-app/evals/reports/triage-question-failure.md +0 -17
- package/evals/fixtures/tiny-eval-method-app/evals/runs/review-pr-with-skill-output.diff +0 -11
- package/evals/fixtures/tiny-eval-method-app/evals/runs/review-pr-with-skill-output.md +0 -3
- package/evals/fixtures/tiny-eval-method-app/skills/review-pr/SKILL.md +0 -10
- package/evals/fixtures/tiny-eval-method-app/skills/triage-question/SKILL.md +0 -10
- package/evals/fixtures/tiny-execute-plan-app/README.md +0 -5
- package/evals/fixtures/tiny-execute-plan-app/checks/settings-and-billing-check.sh +0 -5
- package/evals/fixtures/tiny-execute-plan-app/docs/billing-policy.md +0 -10
- package/evals/fixtures/tiny-execute-plan-app/docs/plans/immediate-downgrade-plan.md +0 -11
- package/evals/fixtures/tiny-execute-plan-app/docs/plans/settings-title-plan.md +0 -12
- package/evals/fixtures/tiny-execute-plan-app/docs/plans/settings-title-with-bad-check.md +0 -15
- package/evals/fixtures/tiny-execute-plan-app/docs/plans/unverified-billing-plan.md +0 -9
- package/evals/fixtures/tiny-execute-plan-app/docs/specs/failed-payment-grace-period.md +0 -9
- package/evals/fixtures/tiny-execute-plan-app/docs/specs/settings-title.md +0 -9
- package/evals/fixtures/tiny-execute-plan-app/src/billing.ts +0 -7
- package/evals/fixtures/tiny-execute-plan-app/src/settings-page.ts +0 -1
- package/evals/fixtures/tiny-execute-plan-app/tests/billing.test.md +0 -7
- package/evals/fixtures/tiny-execute-plan-app/tests/settings.test.md +0 -5
- package/evals/fixtures/tiny-execute-review-loop-app/README.md +0 -5
- package/evals/fixtures/tiny-execute-review-loop-app/docs/billing-policy.md +0 -9
- package/evals/fixtures/tiny-execute-review-loop-app/docs/plans/settings-title-plan.md +0 -12
- package/evals/fixtures/tiny-execute-review-loop-app/docs/review/previous-agent-summary.md +0 -15
- package/evals/fixtures/tiny-execute-review-loop-app/docs/review/review-pass-1-non-pass.md +0 -11
- package/evals/fixtures/tiny-execute-review-loop-app/docs/specs/settings-title.md +0 -9
- package/evals/fixtures/tiny-execute-review-loop-app/src/billing.ts +0 -7
- package/evals/fixtures/tiny-execute-review-loop-app/src/settings-page.ts +0 -1
- package/evals/fixtures/tiny-execute-review-loop-app/tests/billing.test.md +0 -7
- package/evals/fixtures/tiny-execute-review-loop-app/tests/settings.test.md +0 -5
- package/evals/fixtures/tiny-flaky-session-app/README.md +0 -10
- package/evals/fixtures/tiny-flaky-session-app/src/session-timeout.ts +0 -13
- package/evals/fixtures/tiny-flaky-session-app/tests/session-timeout.test.md +0 -8
- package/evals/fixtures/tiny-handoff-app/README.md +0 -5
- package/evals/fixtures/tiny-handoff-app/docs/billing-policy.md +0 -10
- package/evals/fixtures/tiny-handoff-app/docs/handoffs/old-agent-handoff.md +0 -8
- package/evals/fixtures/tiny-handoff-app/docs/plans/billing-retry-plan.md +0 -11
- package/evals/fixtures/tiny-handoff-app/src/billing.ts +0 -7
- package/evals/fixtures/tiny-handoff-report-app/README.md +0 -3
- package/evals/fixtures/tiny-handoff-report-app/evals/runs/handoff-3/hof-003/with-skill-output.diff +0 -1
- package/evals/fixtures/tiny-handoff-report-app/evals/runs/handoff-3/hof-003/with-skill-output.md +0 -1
- package/evals/fixtures/tiny-handoff-report-app/evals/scores/hof-003-score.txt +0 -4
- package/evals/fixtures/tiny-plan-app/README.md +0 -5
- package/evals/fixtures/tiny-plan-app/docs/billing-policy.md +0 -10
- package/evals/fixtures/tiny-plan-app/docs/specs/failed-payment-grace-period.md +0 -25
- package/evals/fixtures/tiny-plan-app/src/billing.ts +0 -17
- package/evals/fixtures/tiny-plan-app/src/settings-page.ts +0 -1
- package/evals/fixtures/tiny-plan-app/tests/billing.test.md +0 -8
- package/evals/fixtures/tiny-plan-app/tests/settings.test.md +0 -5
- package/evals/fixtures/tiny-plan-strict-api-app/README.md +0 -3
- package/evals/fixtures/tiny-plan-strict-api-app/docs/billing-policy.md +0 -5
- package/evals/fixtures/tiny-plan-strict-api-app/docs/specs/billing-webhook-api.md +0 -25
- package/evals/fixtures/tiny-plan-strict-api-app/src/billing.ts +0 -5
- package/evals/fixtures/tiny-plan-strict-api-app/src/webhooks.ts +0 -3
- package/evals/fixtures/tiny-plan-strict-api-app/tests/billing-webhook.test.md +0 -8
- package/evals/fixtures/tiny-post-setup-invalid-config-app/.freeflow/config.json +0 -3
- package/evals/fixtures/tiny-post-setup-invalid-config-app/AGENTS.md +0 -15
- package/evals/fixtures/tiny-post-setup-invalid-config-app/README.md +0 -16
- package/evals/fixtures/tiny-post-setup-invalid-config-app/docs/auth-review.md +0 -15
- package/evals/fixtures/tiny-post-setup-invalid-config-app/docs/billing-policy.md +0 -10
- package/evals/fixtures/tiny-post-setup-invalid-config-app/docs/copy-conventions.md +0 -15
- package/evals/fixtures/tiny-post-setup-invalid-config-app/src/auth-middleware.ts +0 -12
- package/evals/fixtures/tiny-post-setup-invalid-config-app/src/billing.ts +0 -20
- package/evals/fixtures/tiny-post-setup-invalid-config-app/src/login-copy.ts +0 -1
- package/evals/fixtures/tiny-post-setup-invalid-config-app/src/settings-page.ts +0 -1
- package/evals/fixtures/tiny-post-setup-invalid-config-app/tests/auth-middleware.test.md +0 -8
- package/evals/fixtures/tiny-post-setup-invalid-config-app/tests/billing.test.md +0 -8
- package/evals/fixtures/tiny-post-setup-invalid-config-app/tests/copy.test.md +0 -7
- package/evals/fixtures/tiny-post-setup-missing-config-app/AGENTS.md +0 -15
- package/evals/fixtures/tiny-post-setup-missing-config-app/README.md +0 -16
- package/evals/fixtures/tiny-post-setup-missing-config-app/docs/auth-review.md +0 -15
- package/evals/fixtures/tiny-post-setup-missing-config-app/docs/billing-policy.md +0 -10
- package/evals/fixtures/tiny-post-setup-missing-config-app/docs/copy-conventions.md +0 -15
- package/evals/fixtures/tiny-post-setup-missing-config-app/src/auth-middleware.ts +0 -12
- package/evals/fixtures/tiny-post-setup-missing-config-app/src/billing.ts +0 -20
- package/evals/fixtures/tiny-post-setup-missing-config-app/src/login-copy.ts +0 -1
- package/evals/fixtures/tiny-post-setup-missing-config-app/src/settings-page.ts +0 -1
- package/evals/fixtures/tiny-post-setup-missing-config-app/tests/auth-middleware.test.md +0 -8
- package/evals/fixtures/tiny-post-setup-missing-config-app/tests/billing.test.md +0 -8
- package/evals/fixtures/tiny-post-setup-missing-config-app/tests/copy.test.md +0 -7
- package/evals/fixtures/tiny-post-setup-source-conflict-app/.freeflow/config.json +0 -3
- package/evals/fixtures/tiny-post-setup-source-conflict-app/AGENTS.md +0 -15
- package/evals/fixtures/tiny-post-setup-source-conflict-app/README.md +0 -16
- package/evals/fixtures/tiny-post-setup-source-conflict-app/docs/auth-review.md +0 -15
- package/evals/fixtures/tiny-post-setup-source-conflict-app/docs/billing-policy.md +0 -10
- package/evals/fixtures/tiny-post-setup-source-conflict-app/docs/copy-conventions.md +0 -15
- package/evals/fixtures/tiny-post-setup-source-conflict-app/src/auth-middleware.ts +0 -12
- package/evals/fixtures/tiny-post-setup-source-conflict-app/src/billing.ts +0 -20
- package/evals/fixtures/tiny-post-setup-source-conflict-app/src/login-copy.ts +0 -1
- package/evals/fixtures/tiny-post-setup-source-conflict-app/src/settings-page.ts +0 -1
- package/evals/fixtures/tiny-post-setup-source-conflict-app/tests/auth-middleware.test.md +0 -8
- package/evals/fixtures/tiny-post-setup-source-conflict-app/tests/billing.test.md +0 -8
- package/evals/fixtures/tiny-post-setup-source-conflict-app/tests/copy.test.md +0 -7
- package/evals/fixtures/tiny-report-performance-app/README.md +0 -11
- package/evals/fixtures/tiny-report-performance-app/docs/performance.md +0 -10
- package/evals/fixtures/tiny-report-performance-app/src/report-summary.ts +0 -29
- package/evals/fixtures/tiny-report-performance-app/tests/report-performance.test.md +0 -8
- package/evals/fixtures/tiny-review-request-app/README.md +0 -3
- package/evals/fixtures/tiny-review-request-app/docs/billing-policy.md +0 -5
- package/evals/fixtures/tiny-review-request-app/docs/plans/billing-webhook-api.md +0 -19
- package/evals/fixtures/tiny-review-request-app/docs/review/previous-agent-summary.md +0 -20
- package/evals/fixtures/tiny-review-request-app/docs/specs/billing-webhook-api.md +0 -25
- package/evals/fixtures/tiny-review-request-app/src/billing.ts +0 -11
- package/evals/fixtures/tiny-review-request-app/src/webhooks.ts +0 -19
- package/evals/fixtures/tiny-review-request-app/tests/billing-webhook.test.md +0 -8
- package/evals/fixtures/tiny-review-work-app/README.md +0 -5
- package/evals/fixtures/tiny-review-work-app/docs/billing-policy.md +0 -9
- package/evals/fixtures/tiny-review-work-app/docs/plans/settings-title-plan.md +0 -12
- package/evals/fixtures/tiny-review-work-app/docs/review/previous-agent-summary.md +0 -15
- package/evals/fixtures/tiny-review-work-app/docs/specs/settings-title.md +0 -9
- package/evals/fixtures/tiny-review-work-app/src/billing.ts +0 -7
- package/evals/fixtures/tiny-review-work-app/src/settings-page.ts +0 -1
- package/evals/fixtures/tiny-review-work-app/tests/billing.test.md +0 -7
- package/evals/fixtures/tiny-review-work-app/tests/settings.test.md +0 -5
- package/evals/fixtures/tiny-saas-app/README.md +0 -16
- package/evals/fixtures/tiny-saas-app/docs/auth-review.md +0 -15
- package/evals/fixtures/tiny-saas-app/docs/billing-policy.md +0 -10
- package/evals/fixtures/tiny-saas-app/docs/copy-conventions.md +0 -15
- package/evals/fixtures/tiny-saas-app/docs/specs/settings-title.md +0 -13
- package/evals/fixtures/tiny-saas-app/src/auth-middleware.ts +0 -12
- package/evals/fixtures/tiny-saas-app/src/billing.ts +0 -20
- package/evals/fixtures/tiny-saas-app/src/login-copy.ts +0 -1
- package/evals/fixtures/tiny-saas-app/src/settings-page.ts +0 -1
- package/evals/fixtures/tiny-saas-app/tests/auth-middleware.test.md +0 -8
- package/evals/fixtures/tiny-saas-app/tests/billing.test.md +0 -8
- package/evals/fixtures/tiny-saas-app/tests/copy.test.md +0 -7
- package/evals/fixtures/tiny-setup-agents-app/AGENTS.md +0 -6
- package/evals/fixtures/tiny-setup-both-app/AGENTS.md +0 -5
- package/evals/fixtures/tiny-setup-both-app/CLAUDE.md +0 -5
- package/evals/fixtures/tiny-setup-claude-app/CLAUDE.md +0 -6
- package/evals/fixtures/tiny-setup-codex-rules-app/.codex/rules/shell.rules +0 -5
- package/evals/fixtures/tiny-setup-codex-rules-app/AGENTS.md +0 -6
- package/evals/fixtures/tiny-setup-conflict-app/AGENTS.md +0 -6
- package/evals/fixtures/tiny-setup-existing-block-app/AGENTS.md +0 -14
- package/evals/fixtures/tiny-setup-four-rules-app/CLAUDE.md +0 -6
- package/evals/fixtures/tiny-skill-dev-app/README.md +0 -10
- package/evals/fixtures/tiny-skill-dev-app/evals/prompts/review-pr-001.txt +0 -8
- package/evals/fixtures/tiny-skill-dev-app/evals/reports/review-pr-failure.md +0 -17
- package/evals/fixtures/tiny-skill-dev-app/skills/review-pr/SKILL.md +0 -13
- package/evals/fixtures/tiny-team-app/README.md +0 -5
- package/evals/fixtures/tiny-team-app/docs/product.md +0 -6
- package/evals/fixtures/tiny-team-app/src/teams.ts +0 -9
- package/evals/fixtures/tiny-team-app/src/users.ts +0 -5
- package/evals/fixtures/tiny-team-app/tests/teams.test.md +0 -7
- package/evals/fixtures/tiny-verify-app/README.md +0 -5
- package/evals/fixtures/tiny-verify-app/src/settings-nav.ts +0 -1
- package/evals/fixtures/tiny-verify-app/src/settings-page.ts +0 -1
- package/evals/fixtures/tiny-verify-app/tests/settings.test.md +0 -7
- package/evals/prompts/aon-001.txt +0 -2
- package/evals/prompts/byp-001.txt +0 -2
- package/evals/prompts/byp-002.txt +0 -2
- package/evals/prompts/cap-001.txt +0 -3
- package/evals/prompts/cap-002.txt +0 -3
- package/evals/prompts/cmd-001.txt +0 -3
- package/evals/prompts/cmd-002.txt +0 -3
- package/evals/prompts/cmd-003.txt +0 -6
- package/evals/prompts/cmd-004.txt +0 -5
- package/evals/prompts/cmd-005.txt +0 -5
- package/evals/prompts/cmd-006.txt +0 -5
- package/evals/prompts/cmd-007.txt +0 -5
- package/evals/prompts/cmd-008.txt +0 -5
- package/evals/prompts/cmd-009.txt +0 -8
- package/evals/prompts/cmd-010.txt +0 -5
- package/evals/prompts/cmd-011.txt +0 -5
- package/evals/prompts/cmd-012.txt +0 -5
- package/evals/prompts/cmd-013.txt +0 -5
- package/evals/prompts/cmd-014.txt +0 -7
- package/evals/prompts/cmd-015.txt +0 -7
- package/evals/prompts/cmd-016.txt +0 -5
- package/evals/prompts/cmt-001.txt +0 -1
- package/evals/prompts/cmt-002.txt +0 -1
- package/evals/prompts/cmt-003.txt +0 -1
- package/evals/prompts/cmt-004.txt +0 -1
- package/evals/prompts/cmt-005.txt +0 -1
- package/evals/prompts/dfd-001.txt +0 -3
- package/evals/prompts/dia-001.txt +0 -3
- package/evals/prompts/dia-002.txt +0 -3
- package/evals/prompts/dia-003.txt +0 -3
- package/evals/prompts/dis-001.txt +0 -5
- package/evals/prompts/dis-002.txt +0 -5
- package/evals/prompts/esk-001.txt +0 -5
- package/evals/prompts/esk-002.txt +0 -5
- package/evals/prompts/esk-003.txt +0 -5
- package/evals/prompts/grc-001.txt +0 -3
- package/evals/prompts/hof-001.txt +0 -3
- package/evals/prompts/hof-002.txt +0 -1
- package/evals/prompts/hof-003.txt +0 -1
- package/evals/prompts/hof-004.txt +0 -1
- package/evals/prompts/hof-005.txt +0 -1
- package/evals/prompts/ivg-001.txt +0 -1
- package/evals/prompts/ivg-002.txt +0 -11
- package/evals/prompts/ivg-003.txt +0 -1
- package/evals/prompts/ivg-004.txt +0 -1
- package/evals/prompts/mode-001.txt +0 -3
- package/evals/prompts/mode-002.txt +0 -1
- package/evals/prompts/mode-003.txt +0 -3
- package/evals/prompts/mode-004.txt +0 -3
- package/evals/prompts/mode-005.txt +0 -3
- package/evals/prompts/mode-006.txt +0 -3
- package/evals/prompts/mode-007.txt +0 -3
- package/evals/prompts/otr-001.txt +0 -3
- package/evals/prompts/otr-002.txt +0 -3
- package/evals/prompts/rar-001.txt +0 -5
- package/evals/prompts/rar-002.txt +0 -3
- package/evals/prompts/rar-003.txt +0 -6
- package/evals/prompts/rar-004.txt +0 -15
- package/evals/prompts/rar-005.txt +0 -10
- package/evals/prompts/rbr-001.txt +0 -3
- package/evals/prompts/rbr-002.txt +0 -3
- package/evals/prompts/rbr-003.txt +0 -7
- package/evals/prompts/rev-001.txt +0 -3
- package/evals/prompts/rev-002.txt +0 -5
- package/evals/prompts/rev-003.txt +0 -6
- package/evals/prompts/rev-004.txt +0 -1
- package/evals/prompts/rev-005.txt +0 -15
- package/evals/prompts/rev-006.txt +0 -5
- package/evals/prompts/stp-001.txt +0 -1
- package/evals/prompts/stp-002.txt +0 -1
- package/evals/prompts/stp-003.txt +0 -1
- package/evals/prompts/stp-004.txt +0 -1
- package/evals/prompts/stp-005.txt +0 -1
- package/evals/prompts/stp-006.txt +0 -1
- package/evals/prompts/stp-007.txt +0 -1
- package/evals/prompts/stp-008.txt +0 -1
- package/evals/prompts/stp-009.txt +0 -1
- package/evals/prompts/stp-010.txt +0 -1
- package/evals/prompts/stp-011.txt +0 -1
- package/evals/prompts/vfy-003.txt +0 -1
- package/evals/prompts/wpl-001.txt +0 -3
- package/evals/prompts/wpl-002.txt +0 -5
- package/evals/prompts/wpl-003.txt +0 -3
- package/evals/prompts/wpl-004.txt +0 -5
- package/evals/prompts/wpl-005.txt +0 -1
- package/evals/prompts/wrc-001.txt +0 -10
- package/evals/prompts/wrc-002.txt +0 -3
- package/evals/prompts/wsk-001.txt +0 -6
- package/evals/prompts/wsp-001.txt +0 -5
- package/evals/prompts/wsp-002.txt +0 -15
- package/evals/prompts/wsp-003.txt +0 -3
- package/evals/prompts/wsp-004.txt +0 -15
- package/evals/prompts/wsp-005.txt +0 -5
- package/evals/prompts/wsp-006.txt +0 -12
- package/evals/prompts/wsp-007.txt +0 -9
- package/evals/prompts/wsp-008.txt +0 -13
- package/evals/prompts/xpl-001.txt +0 -1
- package/evals/prompts/xpl-002.txt +0 -3
- package/evals/prompts/xpl-003.txt +0 -3
- package/evals/prompts/xpl-004.txt +0 -3
- package/evals/prompts/xpl-005.txt +0 -5
- package/evals/prompts/xpl-006.txt +0 -3
- package/evals/registries/adversarial-evals.json +0 -39
- package/evals/registries/fixture-evals.json +0 -2513
- package/evals/registries/handoff-evals.json +0 -74
- package/evals/registries/legacy-evals.json +0 -147
- package/evals/registries/skill-evidence.json +0 -193
- package/evals/registries/verify-work-evals.json +0 -46
- package/evals/reports/acceptance/v0.1-acceptance-report.md +0 -130
- package/evals/reports/by-command-surface/command-surface-1-report.md +0 -45
- package/evals/reports/by-command-surface/command-surface-10-report.md +0 -76
- package/evals/reports/by-command-surface/command-surface-11-report.md +0 -77
- package/evals/reports/by-command-surface/command-surface-12-report.md +0 -91
- package/evals/reports/by-command-surface/command-surface-13-report.md +0 -64
- package/evals/reports/by-command-surface/command-surface-14-report.md +0 -66
- package/evals/reports/by-command-surface/command-surface-15-report.md +0 -124
- package/evals/reports/by-command-surface/command-surface-16-report.md +0 -39
- package/evals/reports/by-command-surface/command-surface-2-report.md +0 -80
- package/evals/reports/by-command-surface/command-surface-3-report.md +0 -86
- package/evals/reports/by-command-surface/command-surface-4-report.md +0 -76
- package/evals/reports/by-command-surface/command-surface-5-report.md +0 -80
- package/evals/reports/by-command-surface/command-surface-6-report.md +0 -74
- package/evals/reports/by-command-surface/command-surface-7-report.md +0 -84
- package/evals/reports/by-command-surface/command-surface-8-report.md +0 -79
- package/evals/reports/by-command-surface/command-surface-9-report.md +0 -79
- package/evals/reports/by-command-surface/command-surface-matrix.md +0 -91
- package/evals/reports/by-skill/bypass-1-report.md +0 -124
- package/evals/reports/by-skill/capture-decisions-1-report.md +0 -99
- package/evals/reports/by-skill/capture-decisions-2-report.md +0 -102
- package/evals/reports/by-skill/commit-work-1-report.md +0 -165
- package/evals/reports/by-skill/commit-work-2-report.md +0 -109
- package/evals/reports/by-skill/design-for-depth-1-report.md +0 -95
- package/evals/reports/by-skill/diagnose-failure-1-report.md +0 -91
- package/evals/reports/by-skill/diagnose-failure-2-report.md +0 -94
- package/evals/reports/by-skill/discover-1-report.md +0 -118
- package/evals/reports/by-skill/evaluate-skill-2-report.md +0 -71
- package/evals/reports/by-skill/execute-plan-1-report.md +0 -103
- package/evals/reports/by-skill/execute-plan-2-report.md +0 -112
- package/evals/reports/by-skill/execute-plan-3-report.md +0 -87
- package/evals/reports/by-skill/grill-context-1-report.md +0 -103
- package/evals/reports/by-skill/handoff-1-report.md +0 -123
- package/evals/reports/by-skill/handoff-2-report.md +0 -105
- package/evals/reports/by-skill/handoff-3-report.md +0 -100
- package/evals/reports/by-skill/handoff-7-report.md +0 -77
- package/evals/reports/by-skill/interview-gate-1-report.md +0 -63
- package/evals/reports/by-skill/interview-gate-2-report.md +0 -105
- package/evals/reports/by-skill/mode-contract-1-report.md +0 -74
- package/evals/reports/by-skill/mode-contract-2-report.md +0 -87
- package/evals/reports/by-skill/mode-contract-3-report.md +0 -65
- package/evals/reports/by-skill/mode-contract-4-report.md +0 -71
- package/evals/reports/by-skill/mode-contract-5-report.md +0 -70
- package/evals/reports/by-skill/output-router-1-report.md +0 -76
- package/evals/reports/by-skill/output-router-2-report.md +0 -65
- package/evals/reports/by-skill/research-1-report.md +0 -144
- package/evals/reports/by-skill/research-brief-1-report.md +0 -94
- package/evals/reports/by-skill/research-brief-2-report.md +0 -75
- package/evals/reports/by-skill/review-artifact-1-report.md +0 -91
- package/evals/reports/by-skill/review-artifact-2-report.md +0 -79
- package/evals/reports/by-skill/review-artifact-3-report.md +0 -120
- package/evals/reports/by-skill/review-artifact-4-report.md +0 -89
- package/evals/reports/by-skill/review-artifact-5-report.md +0 -66
- package/evals/reports/by-skill/review-work-1-report.md +0 -85
- package/evals/reports/by-skill/review-work-2-report.md +0 -87
- package/evals/reports/by-skill/review-work-3-report.md +0 -104
- package/evals/reports/by-skill/review-work-4-report.md +0 -108
- package/evals/reports/by-skill/review-work-5-report.md +0 -73
- package/evals/reports/by-skill/review-work-6-report.md +0 -66
- package/evals/reports/by-skill/setup-freeflow-1-report.md +0 -199
- package/evals/reports/by-skill/setup-freeflow-2-report.md +0 -105
- package/evals/reports/by-skill/setup-freeflow-3-report.md +0 -105
- package/evals/reports/by-skill/setup-freeflow-4-report.md +0 -88
- package/evals/reports/by-skill/setup-freeflow-5-report.md +0 -115
- package/evals/reports/by-skill/skill-authoring-1-report.md +0 -81
- package/evals/reports/by-skill/verify-work-1-report.md +0 -111
- package/evals/reports/by-skill/verify-work-2-report.md +0 -120
- package/evals/reports/by-skill/workflow-1-report.md +0 -65
- package/evals/reports/by-skill/write-plan-1-report.md +0 -123
- package/evals/reports/by-skill/write-plan-2-report.md +0 -98
- package/evals/reports/by-skill/write-plan-3-report.md +0 -110
- package/evals/reports/by-skill/write-spec-1-report.md +0 -88
- package/evals/reports/by-skill/write-spec-2-report.md +0 -89
- package/evals/reports/by-skill/write-spec-3-report.md +0 -90
- package/evals/reports/by-skill/write-spec-4-report.md +0 -167
- package/evals/reports/harness/claude-harness-1-report.md +0 -103
- package/evals/reports/harness/eval-harness-1-report.md +0 -91
- package/evals/reports/iterations/iteration-1-report.md +0 -71
- package/evals/reports/iterations/iteration-2-report.md +0 -78
- package/evals/reports/iterations/iteration-3-report.md +0 -118
- package/evals/reports/iterations/iteration-4-report.md +0 -131
- package/evals/reports/iterations/iteration-5-report.md +0 -149
- package/evals/reports/iterations/iteration-6-report.md +0 -136
- package/evals/reports/runtime/always-on-runtime-1-report.md +0 -107
- package/evals/reports/runtime/context-mode-normalized-benchmark-1-report.md +0 -52
- package/evals/reports/runtime/context-mode-real-deep-baseline-1-report.md +0 -410
- package/evals/reports/runtime/context-mode-real-deep-final-slice-11-report.md +0 -203
- package/evals/reports/runtime/context-mode-real-deep-final-slice-11-review.md +0 -51
- package/evals/reports/runtime/context-saving-native-tools-baseline-1-report.md +0 -99
- package/evals/reports/runtime/eryx-python-compatibility-reprobe-1-report.md +0 -166
- package/evals/reports/runtime/eryx-python-proof-spike-1-report.md +0 -96
- package/evals/reports/runtime/eryx-python-proof-spike-2-report.md +0 -151
- package/evals/reports/runtime/eryx-python-script-derive-execution-1-report.md +0 -233
- package/evals/reports/runtime/jq-script-derive-execution-1-report.md +0 -215
- package/evals/reports/runtime/jq-wasm-proof-spike-1-report.md +0 -67
- package/evals/reports/runtime/output-router-benchmark-1-report.md +0 -81
- package/evals/reports/runtime/output-router-capture-derive-provider-eval-1-report.md +0 -35
- package/evals/reports/runtime/output-router-codex-large-index-sqlite-adhoc-report.md +0 -150
- package/evals/reports/runtime/output-router-codex-qa-benchmark-1-report.md +0 -46
- package/evals/reports/runtime/output-router-command-benchmark-1-report.md +0 -62
- package/evals/reports/runtime/output-router-index-benchmark-1-report.md +0 -64
- package/evals/reports/runtime/output-router-regression-1-report.md +0 -81
- package/evals/reports/runtime/output-router-transform-eval-1-report.md +0 -28
- package/evals/reports/runtime/pi-observed-routing-eval-1-report.md +0 -44
- package/evals/reports/runtime/quickjs-script-derive-execution-1-report.md +0 -159
- package/evals/reports/runtime/quickjs-wasi-proof-spike-1-report.md +0 -55
- package/evals/reports/runtime/render-format-row-protocol-1-report.md +0 -58
- package/evals/reports/runtime/script-sandbox-adapter-selection-review-1-report.md +0 -84
- package/evals/reports/runtime/script-sandbox-probe-resource-hardening-1-report.md +0 -51
- package/evals/reports/runtime/storage-policy-benchmark-1-report.md +0 -176
- package/evals/reports/runtime/vault-index-storage-spike-1-report.md +0 -39
- package/evals/reports/runtime/workflow-context-hook-1-report.md +0 -82
- package/evals/runbooks/adversarial-eval-runbook.md +0 -28
- package/evals/runbooks/fixture-eval-runbook.md +0 -137
- package/evals/runbooks/handoff-eval-runbook.md +0 -22
- package/evals/runbooks/verify-work-eval-runbook.md +0 -27
- package/evals/scripts/audit-command-surface.sh +0 -160
- package/evals/scripts/check-activation-contract.sh +0 -121
- package/evals/scripts/check-runtime-context-hook.sh +0 -183
- package/evals/scripts/grade-fixture-eval.sh +0 -446
- package/evals/scripts/run-claude-fixture-eval.sh +0 -26
- package/evals/scripts/run-codex-fixture-eval.sh +0 -26
- package/evals/scripts/run-eryx-python-proof-spike.js +0 -559
- package/evals/scripts/run-fixture-eval-by-id.sh +0 -23
- package/evals/scripts/run-fixture-eval.sh +0 -464
- package/evals/scripts/run-jq-wasm-proof-spike.js +0 -299
- package/evals/scripts/run-output-router-transform-eval.js +0 -115
- package/evals/scripts/run-pi-observed-routing-eval.js +0 -478
- package/evals/scripts/run-quickjs-wasi-proof-spike.js +0 -339
- package/evals/scripts/skill-evidence.sh +0 -316
- package/evals/scripts/validate-release-metadata.sh +0 -486
- package/evals/suites/v0.1-acceptance-suite.md +0 -131
- package/pi-extension/dist/host-producer-identification.js +0 -186
- package/pi-extension/dist/mcp-config.js +0 -30
- package/pi-extension/dist/native-safety-net.js +0 -209
- package/pi-extension/dist/observed-tool-routing.js +0 -114
- package/pi-extension/dist/renderers.js +0 -835
- package/pi-extension/dist/router-tools.js +0 -438
- package/pi-extension/dist/runtime-context.js +0 -288
- package/pi-extension/dist/schemas.js +0 -254
- package/pi-extension/dist/status.js +0 -449
- package/pi-extension/dist/utils.js +0 -467
- package/pi-extension/src/runtime-context.ts +0 -347
- package/pi-extension/src/status.ts +0 -505
- package/plugin-docs/README.md +0 -40
- package/plugin-docs/adr/0001-three-modes.md +0 -27
- package/plugin-docs/adr/0002-evals-before-hooks.md +0 -17
- package/plugin-docs/adr/0003-release-boundary.md +0 -21
- package/plugin-docs/adr/0004-discover-replaces-shallow-discovery-skills.md +0 -26
- package/plugin-docs/adr/README.md +0 -10
- package/plugin-docs/architecture.md +0 -89
- package/plugin-docs/output-router.md +0 -396
- package/plugin-docs/release-evidence.md +0 -77
- package/plugin-docs/skills.md +0 -44
- package/plugin-docs/workflow.md +0 -57
- package/skills/diagnose-failure/references/feedback-loop-catalog.md +0 -25
- package/skills/discover/ARTIFACT-DESTINATIONS.md +0 -58
- package/skills/discover/CHECKPOINTS.md +0 -74
- package/skills/discover/SKILL.md +0 -173
- package/skills/evaluate-skill/references/eval-patterns.md +0 -41
- package/skills/evaluate-skill/references/grading-priority.md +0 -33
- package/skills/execute-plan/SKILL.md +0 -133
- package/skills/execute-plan/references/execution-map.md +0 -122
- package/skills/interview-gate/SKILL.md +0 -90
- package/skills/setup-freeflow/references/output-router-setup.md +0 -102
- package/skills/workflow/references/workflow-map.md +0 -104
- /package/pi-extension/src/{mcp-config.ts → integrations/mcp-config.ts} +0 -0
|
@@ -2,25 +2,37 @@ import { createHash } from "node:crypto";
|
|
|
2
2
|
import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
|
|
3
3
|
import { tmpdir } from "node:os";
|
|
4
4
|
import { join } from "node:path";
|
|
5
|
-
import {
|
|
5
|
+
import {
|
|
6
|
+
DEFAULT_ROUTER_THRESHOLDS,
|
|
7
|
+
DEFAULT_SCRIPT_TRANSFORM_CONFIG,
|
|
8
|
+
MAX_SCRIPT_TRANSFORM_LIMITS,
|
|
9
|
+
SCRIPT_TRANSFORM_LANGUAGES,
|
|
10
|
+
} from "../config/config.js";
|
|
6
11
|
import { assembleTextEvidence, byteLength, countLines, splitLines } from "../evidence/evidence.js";
|
|
7
12
|
import { selectScriptSandboxAdapter } from "../sandbox/script-sandbox.js";
|
|
8
|
-
import {
|
|
13
|
+
import {
|
|
14
|
+
transformAdapterUnavailableFailure,
|
|
15
|
+
transformSourceUnavailableFailure,
|
|
16
|
+
transformValidationFailure,
|
|
17
|
+
transformExecutionFailure,
|
|
18
|
+
scriptTransformDisabledFailure,
|
|
19
|
+
storageFailure,
|
|
20
|
+
} from "../evidence/failure-contracts.js";
|
|
9
21
|
import { createVault, readOutputText, readVaultRecord, storeTextOutput } from "../vault/vault.js";
|
|
10
22
|
export const TRANSFORM_ENGINE_IMPLEMENTATION = "shared-transform-engine-v1";
|
|
11
23
|
const PRESERVE_MODES = new Set(["summary", "important", "full"]);
|
|
12
24
|
const OPERATION_KINDS = new Set([
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
25
|
+
"regexFilter",
|
|
26
|
+
"countMatches",
|
|
27
|
+
"jsonExtract",
|
|
28
|
+
"groupByRegex",
|
|
29
|
+
"dedupe",
|
|
30
|
+
"topN",
|
|
31
|
+
"extractUrls",
|
|
32
|
+
"extractCitations",
|
|
33
|
+
"lineStats",
|
|
34
|
+
"sizeStats",
|
|
35
|
+
"script",
|
|
24
36
|
]);
|
|
25
37
|
const REGEX_FLAGS = new Set(["g", "i", "m", "s", "u"]);
|
|
26
38
|
const DEFAULT_CONTEXT_LINES = 0;
|
|
@@ -40,1916 +52,2021 @@ const URL_PATTERN = /https?:\/\/[^\s<>"'\])}]+/gi;
|
|
|
40
52
|
const SCRIPT_ALIAS_PATTERN = /^[A-Za-z][A-Za-z0-9_-]{0,63}$/;
|
|
41
53
|
const MAX_SCRIPT_CODE_BYTES = 256 * 1024;
|
|
42
54
|
export function validateTransformInput(value) {
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
if (operationKind === "script") {
|
|
63
|
-
const input = {
|
|
64
|
-
sources: value.sources,
|
|
65
|
-
operation: value.operation,
|
|
66
|
-
};
|
|
67
|
-
if (value.limits !== undefined) {
|
|
68
|
-
input.limits = value.limits;
|
|
69
|
-
}
|
|
70
|
-
if (value.preserve !== undefined) {
|
|
71
|
-
input.preserve = value.preserve;
|
|
72
|
-
}
|
|
73
|
-
return { ok: true, value: input };
|
|
74
|
-
}
|
|
55
|
+
const issues = [];
|
|
56
|
+
if (!isRecord(value)) {
|
|
57
|
+
return { ok: false, issues: [{ path: "$", message: "Expected transform input object." }] };
|
|
58
|
+
}
|
|
59
|
+
const operationKind = isRecord(value.operation) ? value.operation.kind : undefined;
|
|
60
|
+
validateTransformOperation(value.operation, "$.operation", issues);
|
|
61
|
+
if (operationKind === "script") {
|
|
62
|
+
validateScriptTransformSources(value.sources, "$.sources", issues);
|
|
63
|
+
validateScriptTransformLimits(value.limits, "$.limits", issues);
|
|
64
|
+
} else {
|
|
65
|
+
validateTransformSource(value.source, "$.source", issues);
|
|
66
|
+
}
|
|
67
|
+
if (value.preserve !== undefined && (typeof value.preserve !== "string" || !PRESERVE_MODES.has(value.preserve))) {
|
|
68
|
+
issues.push({ path: "$.preserve", message: "Expected preserve mode summary, important, or full." });
|
|
69
|
+
}
|
|
70
|
+
if (issues.length > 0) {
|
|
71
|
+
return { ok: false, issues };
|
|
72
|
+
}
|
|
73
|
+
if (operationKind === "script") {
|
|
75
74
|
const input = {
|
|
76
|
-
|
|
77
|
-
|
|
75
|
+
sources: value.sources,
|
|
76
|
+
operation: value.operation,
|
|
78
77
|
};
|
|
78
|
+
if (value.limits !== undefined) {
|
|
79
|
+
input.limits = value.limits;
|
|
80
|
+
}
|
|
79
81
|
if (value.preserve !== undefined) {
|
|
80
|
-
|
|
82
|
+
input.preserve = value.preserve;
|
|
81
83
|
}
|
|
82
84
|
return { ok: true, value: input };
|
|
85
|
+
}
|
|
86
|
+
const input = {
|
|
87
|
+
source: value.source,
|
|
88
|
+
operation: value.operation,
|
|
89
|
+
};
|
|
90
|
+
if (value.preserve !== undefined) {
|
|
91
|
+
input.preserve = value.preserve;
|
|
92
|
+
}
|
|
93
|
+
return { ok: true, value: input };
|
|
83
94
|
}
|
|
84
95
|
export async function freeflowTransform(options) {
|
|
85
|
-
|
|
96
|
+
return executeTransform(options);
|
|
86
97
|
}
|
|
87
98
|
async function executeTransform(options) {
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
});
|
|
97
|
-
}
|
|
98
|
-
if (isScriptTransformInput(inputValidation.value)) {
|
|
99
|
-
return handleScriptTransform({
|
|
100
|
-
input: inputValidation.value,
|
|
101
|
-
options,
|
|
102
|
-
preserve,
|
|
103
|
-
});
|
|
104
|
-
}
|
|
105
|
-
const deterministicInput = inputValidation.value;
|
|
106
|
-
const operation = deterministicInput.operation;
|
|
107
|
-
const prepared = prepareTransformOperation(operation);
|
|
108
|
-
if (!prepared.ok) {
|
|
109
|
-
return transformValidationFailureWithOptionalLineage({
|
|
110
|
-
message: prepared.message,
|
|
111
|
-
preserve,
|
|
112
|
-
lineage: lineageFromInput(deterministicInput),
|
|
113
|
-
decisionSeed: "operation-validation",
|
|
114
|
-
});
|
|
115
|
-
}
|
|
116
|
-
const source = deterministicInput.source;
|
|
117
|
-
const vaultOptions = {};
|
|
118
|
-
if (options.vaultRoot !== undefined) {
|
|
119
|
-
vaultOptions.root = options.vaultRoot;
|
|
120
|
-
}
|
|
121
|
-
if (options.vaultRetention !== undefined) {
|
|
122
|
-
vaultOptions.retention = options.vaultRetention;
|
|
123
|
-
}
|
|
124
|
-
const vault = createVault(vaultOptions);
|
|
125
|
-
const thresholds = { ...DEFAULT_ROUTER_THRESHOLDS, ...options.thresholds };
|
|
126
|
-
let sourceRecord;
|
|
127
|
-
let stream;
|
|
128
|
-
let sourceText;
|
|
129
|
-
try {
|
|
130
|
-
sourceRecord = await readVaultRecord(vault, options.sessionId, source.outputId);
|
|
131
|
-
}
|
|
132
|
-
catch (error) {
|
|
133
|
-
return transformSourceUnavailableFailureWithOptionalLineage({
|
|
134
|
-
message: `Vault source outputId=${source.outputId} could not be found or read: ${errorMessage(error)}`,
|
|
135
|
-
preserve,
|
|
136
|
-
lineage: lineageFromInput(inputValidation.value),
|
|
137
|
-
decisionSeed: "source-record",
|
|
138
|
-
});
|
|
139
|
-
}
|
|
140
|
-
const streamResult = resolveSourceStream(sourceRecord, source.stream);
|
|
141
|
-
if (!streamResult.ok) {
|
|
142
|
-
return transformValidationFailure({
|
|
143
|
-
message: streamResult.message,
|
|
144
|
-
preserve,
|
|
145
|
-
lineage: lineageForSource(sourceRecord, operation),
|
|
146
|
-
decisionSeed: "source-stream",
|
|
147
|
-
});
|
|
148
|
-
}
|
|
149
|
-
stream = streamResult.stream;
|
|
150
|
-
try {
|
|
151
|
-
sourceText = await readOutputText(vault, options.sessionId, source.outputId, stream);
|
|
152
|
-
}
|
|
153
|
-
catch (error) {
|
|
154
|
-
return transformSourceUnavailableFailure({
|
|
155
|
-
message: `Vault source outputId=${source.outputId} stream=${stream} could not be read: ${errorMessage(error)}`,
|
|
156
|
-
preserve,
|
|
157
|
-
lineage: lineageForSource(sourceRecord, operation),
|
|
158
|
-
decisionSeed: "source-text",
|
|
159
|
-
});
|
|
160
|
-
}
|
|
161
|
-
let transformed;
|
|
162
|
-
try {
|
|
163
|
-
transformed = transformText({
|
|
164
|
-
text: sourceText,
|
|
165
|
-
sourceLabel: `${source.outputId}:${stream}`,
|
|
166
|
-
operation,
|
|
167
|
-
prepared: prepared.value,
|
|
168
|
-
});
|
|
169
|
-
}
|
|
170
|
-
catch (error) {
|
|
171
|
-
return transformExecutionFailure({
|
|
172
|
-
message: `Transform operation ${operation.kind} failed: ${errorMessage(error)}`,
|
|
173
|
-
preserve,
|
|
174
|
-
lineage: lineageForSource(sourceRecord, operation),
|
|
175
|
-
decisionSeed: "transform-execution",
|
|
176
|
-
});
|
|
177
|
-
}
|
|
178
|
-
const producer = { kind: "transform", name: operation.kind };
|
|
179
|
-
const lineage = lineageForSource(sourceRecord, operation);
|
|
180
|
-
let record;
|
|
181
|
-
try {
|
|
182
|
-
record = await storeTextOutput(vault, {
|
|
183
|
-
sessionId: options.sessionId,
|
|
184
|
-
raw: transformed.text,
|
|
185
|
-
sourceKind: "transform",
|
|
186
|
-
producer,
|
|
187
|
-
lineage,
|
|
188
|
-
decisionIds: [decisionId("transform-store", source.outputId, stream, operation.kind, lineage.operationHash ?? "")],
|
|
189
|
-
});
|
|
190
|
-
}
|
|
191
|
-
catch (error) {
|
|
192
|
-
return storageFailure({
|
|
193
|
-
message: `Transformed output could not be persisted: ${errorMessage(error)}`,
|
|
194
|
-
preserve,
|
|
195
|
-
lineage,
|
|
196
|
-
});
|
|
197
|
-
}
|
|
198
|
-
const routed = routeTransformedText({
|
|
199
|
-
outputId: record.outputId,
|
|
200
|
-
text: transformed.text,
|
|
201
|
-
preserve,
|
|
202
|
-
thresholds,
|
|
203
|
-
source: { kind: "vault", outputId: source.outputId, stream },
|
|
204
|
-
operationKind: operation.kind,
|
|
99
|
+
const preserve = options.preserve ?? "important";
|
|
100
|
+
const inputValidation = validateTransformInput(options);
|
|
101
|
+
if (!inputValidation.ok) {
|
|
102
|
+
return transformValidationFailureWithOptionalLineage({
|
|
103
|
+
message: validationMessage(inputValidation.issues),
|
|
104
|
+
preserve,
|
|
105
|
+
lineage: lineageFromInput(options),
|
|
106
|
+
decisionSeed: "input-validation",
|
|
205
107
|
});
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
108
|
+
}
|
|
109
|
+
if (isScriptTransformInput(inputValidation.value)) {
|
|
110
|
+
return handleScriptTransform({
|
|
111
|
+
input: inputValidation.value,
|
|
112
|
+
options,
|
|
113
|
+
preserve,
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
const deterministicInput = inputValidation.value;
|
|
117
|
+
const operation = deterministicInput.operation;
|
|
118
|
+
const prepared = prepareTransformOperation(operation);
|
|
119
|
+
if (!prepared.ok) {
|
|
120
|
+
return transformValidationFailureWithOptionalLineage({
|
|
121
|
+
message: prepared.message,
|
|
122
|
+
preserve,
|
|
123
|
+
lineage: lineageFromInput(deterministicInput),
|
|
124
|
+
decisionSeed: "operation-validation",
|
|
125
|
+
});
|
|
126
|
+
}
|
|
127
|
+
const source = deterministicInput.source;
|
|
128
|
+
const vaultOptions = {};
|
|
129
|
+
if (options.vaultRoot !== undefined) {
|
|
130
|
+
vaultOptions.root = options.vaultRoot;
|
|
131
|
+
}
|
|
132
|
+
if (options.vaultRetention !== undefined) {
|
|
133
|
+
vaultOptions.retention = options.vaultRetention;
|
|
134
|
+
}
|
|
135
|
+
const vault = createVault(vaultOptions);
|
|
136
|
+
const thresholds = { ...DEFAULT_ROUTER_THRESHOLDS, ...options.thresholds };
|
|
137
|
+
let sourceRecord;
|
|
138
|
+
let stream;
|
|
139
|
+
let sourceText;
|
|
140
|
+
try {
|
|
141
|
+
sourceRecord = await readVaultRecord(vault, options.sessionId, source.outputId);
|
|
142
|
+
} catch (error) {
|
|
143
|
+
return transformSourceUnavailableFailureWithOptionalLineage({
|
|
144
|
+
message: `Vault source outputId=${source.outputId} could not be found or read: ${errorMessage(error)}`,
|
|
145
|
+
preserve,
|
|
146
|
+
lineage: lineageFromInput(inputValidation.value),
|
|
147
|
+
decisionSeed: "source-record",
|
|
148
|
+
});
|
|
149
|
+
}
|
|
150
|
+
const streamResult = resolveSourceStream(sourceRecord, source.stream);
|
|
151
|
+
if (!streamResult.ok) {
|
|
152
|
+
return transformValidationFailure({
|
|
153
|
+
message: streamResult.message,
|
|
154
|
+
preserve,
|
|
155
|
+
lineage: lineageForSource(sourceRecord, operation),
|
|
156
|
+
decisionSeed: "source-stream",
|
|
157
|
+
});
|
|
158
|
+
}
|
|
159
|
+
stream = streamResult.stream;
|
|
160
|
+
try {
|
|
161
|
+
sourceText = await readOutputText(vault, options.sessionId, source.outputId, stream);
|
|
162
|
+
} catch (error) {
|
|
163
|
+
return transformSourceUnavailableFailure({
|
|
164
|
+
message: `Vault source outputId=${source.outputId} stream=${stream} could not be read: ${errorMessage(error)}`,
|
|
165
|
+
preserve,
|
|
166
|
+
lineage: lineageForSource(sourceRecord, operation),
|
|
167
|
+
decisionSeed: "source-text",
|
|
168
|
+
});
|
|
169
|
+
}
|
|
170
|
+
let transformed;
|
|
171
|
+
try {
|
|
172
|
+
transformed = transformText({
|
|
173
|
+
text: sourceText,
|
|
174
|
+
sourceLabel: `${source.outputId}:${stream}`,
|
|
175
|
+
operation,
|
|
176
|
+
prepared: prepared.value,
|
|
177
|
+
});
|
|
178
|
+
} catch (error) {
|
|
179
|
+
return transformExecutionFailure({
|
|
180
|
+
message: `Transform operation ${operation.kind} failed: ${errorMessage(error)}`,
|
|
181
|
+
preserve,
|
|
182
|
+
lineage: lineageForSource(sourceRecord, operation),
|
|
183
|
+
decisionSeed: "transform-execution",
|
|
184
|
+
});
|
|
185
|
+
}
|
|
186
|
+
const producer = { kind: "transform", name: operation.kind };
|
|
187
|
+
const lineage = lineageForSource(sourceRecord, operation);
|
|
188
|
+
let record;
|
|
189
|
+
try {
|
|
190
|
+
record = await storeTextOutput(vault, {
|
|
191
|
+
sessionId: options.sessionId,
|
|
192
|
+
raw: transformed.text,
|
|
193
|
+
sourceKind: "transform",
|
|
194
|
+
producer,
|
|
195
|
+
lineage,
|
|
196
|
+
decisionIds: [
|
|
197
|
+
decisionId("transform-store", source.outputId, stream, operation.kind, lineage.operationHash ?? ""),
|
|
198
|
+
],
|
|
199
|
+
});
|
|
200
|
+
} catch (error) {
|
|
201
|
+
return storageFailure({
|
|
202
|
+
message: `Transformed output could not be persisted: ${errorMessage(error)}`,
|
|
203
|
+
preserve,
|
|
204
|
+
lineage,
|
|
205
|
+
});
|
|
206
|
+
}
|
|
207
|
+
const routed = routeTransformedText({
|
|
208
|
+
outputId: record.outputId,
|
|
209
|
+
text: transformed.text,
|
|
210
|
+
preserve,
|
|
211
|
+
thresholds,
|
|
212
|
+
source: { kind: "vault", outputId: source.outputId, stream },
|
|
213
|
+
operationKind: operation.kind,
|
|
214
|
+
});
|
|
215
|
+
return {
|
|
216
|
+
toolStatus: "ok",
|
|
217
|
+
decisionId: decisionId("transform", record.outputId, source.outputId, stream, operation.kind, routed.routingStatus),
|
|
218
|
+
outputId: record.outputId,
|
|
219
|
+
recordId: record.recordId,
|
|
220
|
+
preserve,
|
|
221
|
+
source: { kind: "vault", outputId: source.outputId, stream },
|
|
222
|
+
operation: operationSummary(operation),
|
|
223
|
+
producer: record.producer,
|
|
224
|
+
persistence: record.persistence,
|
|
225
|
+
lineage,
|
|
226
|
+
routing: {
|
|
227
|
+
status: routed.routingStatus,
|
|
228
|
+
route: "transform",
|
|
229
|
+
reason: routed.reason,
|
|
230
|
+
},
|
|
231
|
+
summary: transformed.summary,
|
|
232
|
+
evidence: routed.evidence,
|
|
233
|
+
recovery: {
|
|
234
|
+
how: `Use freeflow_search with source.kind=vault and outputId=${record.outputId}, stream=raw, and an exact lineRange to recover exact transformed content. Source evidence remains outputId=${source.outputId} stream=${stream}.`,
|
|
235
|
+
outputId: record.outputId,
|
|
236
|
+
},
|
|
237
|
+
};
|
|
229
238
|
}
|
|
230
239
|
async function handleScriptTransform(options) {
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
}
|
|
242
|
-
const vaultOptions = {};
|
|
243
|
-
if (options.options.vaultRoot !== undefined) {
|
|
244
|
-
vaultOptions.root = options.options.vaultRoot;
|
|
245
|
-
}
|
|
246
|
-
if (options.options.vaultRetention !== undefined) {
|
|
247
|
-
vaultOptions.retention = options.options.vaultRetention;
|
|
248
|
-
}
|
|
249
|
-
const vault = createVault(vaultOptions);
|
|
250
|
-
const resolved = await resolveScriptSources({
|
|
251
|
-
vault,
|
|
252
|
-
sessionId: options.options.sessionId,
|
|
253
|
-
sources: options.input.sources,
|
|
254
|
-
maxInputBytes: limits.maxInputBytes,
|
|
255
|
-
operation: options.input.operation,
|
|
256
|
-
preserve: options.preserve,
|
|
240
|
+
const config = effectiveScriptTransformConfig(options.options.scriptTransform);
|
|
241
|
+
const limits = effectiveScriptLimits(config, options.input.limits);
|
|
242
|
+
const initialLineage = lineageForScriptInput(options.input, limits);
|
|
243
|
+
if (!config.enabled) {
|
|
244
|
+
return scriptTransformDisabledFailure({
|
|
245
|
+
message:
|
|
246
|
+
"Script transform is disabled by default. Enable scriptTransform.enabled only after a sandbox adapter has passed capability probes and review. No script code was executed.",
|
|
247
|
+
preserve: options.preserve,
|
|
248
|
+
lineage: initialLineage,
|
|
249
|
+
decisionSeed: "script-transform-disabled",
|
|
257
250
|
});
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
251
|
+
}
|
|
252
|
+
const vaultOptions = {};
|
|
253
|
+
if (options.options.vaultRoot !== undefined) {
|
|
254
|
+
vaultOptions.root = options.options.vaultRoot;
|
|
255
|
+
}
|
|
256
|
+
if (options.options.vaultRetention !== undefined) {
|
|
257
|
+
vaultOptions.retention = options.options.vaultRetention;
|
|
258
|
+
}
|
|
259
|
+
const vault = createVault(vaultOptions);
|
|
260
|
+
const resolved = await resolveScriptSources({
|
|
261
|
+
vault,
|
|
262
|
+
sessionId: options.options.sessionId,
|
|
263
|
+
sources: options.input.sources,
|
|
264
|
+
maxInputBytes: limits.maxInputBytes,
|
|
265
|
+
operation: options.input.operation,
|
|
266
|
+
preserve: options.preserve,
|
|
267
|
+
});
|
|
268
|
+
if (!resolved.ok) {
|
|
269
|
+
return resolved.failure;
|
|
270
|
+
}
|
|
271
|
+
const adapterSelection = await selectScriptSandboxAdapter(
|
|
272
|
+
options.input.operation.language,
|
|
273
|
+
config,
|
|
274
|
+
options.options.scriptSandboxAdapters ?? [],
|
|
275
|
+
);
|
|
276
|
+
if (!adapterSelection.ok) {
|
|
277
|
+
return transformAdapterUnavailableFailure({
|
|
278
|
+
message: `${adapterSelection.status.reason} No script code was executed.`,
|
|
279
|
+
preserve: options.preserve,
|
|
280
|
+
lineage: lineageForResolvedScriptSources(resolved.sources, options.input, limits),
|
|
281
|
+
decisionSeed: "script-transform-adapter-unavailable",
|
|
276
282
|
});
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
vaultOptionsForStore.retention = options.options.vaultRetention;
|
|
293
|
-
}
|
|
294
|
-
const storeVault = createVault(vaultOptionsForStore);
|
|
295
|
-
let record;
|
|
296
|
-
try {
|
|
297
|
-
record = await storeTextOutput(storeVault, {
|
|
298
|
-
sessionId: options.options.sessionId,
|
|
299
|
-
raw: execution.result.stdout,
|
|
300
|
-
sourceKind: "transform",
|
|
301
|
-
producer,
|
|
302
|
-
lineage,
|
|
303
|
-
decisionIds: [decisionId("script-transform-store", options.input.operation.language, lineage.operationHash ?? "")],
|
|
304
|
-
});
|
|
305
|
-
}
|
|
306
|
-
catch (error) {
|
|
307
|
-
return storageFailure({
|
|
308
|
-
message: `Script-transformed output could not be persisted: ${errorMessage(error)}`,
|
|
309
|
-
preserve: options.preserve,
|
|
310
|
-
lineage,
|
|
311
|
-
});
|
|
312
|
-
}
|
|
313
|
-
const thresholds = { ...DEFAULT_ROUTER_THRESHOLDS, ...options.options.thresholds };
|
|
314
|
-
const primarySource = primaryScriptSourceRef(resolved.sources, options.input.sources);
|
|
315
|
-
const routed = routeTransformedText({
|
|
316
|
-
outputId: record.outputId,
|
|
317
|
-
text: execution.result.stdout,
|
|
318
|
-
preserve: options.preserve,
|
|
319
|
-
thresholds,
|
|
320
|
-
source: primarySource,
|
|
321
|
-
operationKind: "script",
|
|
283
|
+
}
|
|
284
|
+
const execution = await executeScriptWithAdapter({
|
|
285
|
+
adapter: adapterSelection.adapter,
|
|
286
|
+
operation: options.input.operation,
|
|
287
|
+
resolvedSources: resolved.sources,
|
|
288
|
+
limits,
|
|
289
|
+
config,
|
|
290
|
+
});
|
|
291
|
+
const lineage = lineageForResolvedScriptSources(resolved.sources, options.input, limits);
|
|
292
|
+
if (!execution.ok) {
|
|
293
|
+
return transformExecutionFailure({
|
|
294
|
+
message: execution.message,
|
|
295
|
+
preserve: options.preserve,
|
|
296
|
+
lineage,
|
|
297
|
+
decisionSeed: "script-transform-execution",
|
|
322
298
|
});
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
299
|
+
}
|
|
300
|
+
const producer = { kind: "transform", name: `script:${options.input.operation.language}` };
|
|
301
|
+
const vaultOptionsForStore = {};
|
|
302
|
+
if (options.options.vaultRoot !== undefined) {
|
|
303
|
+
vaultOptionsForStore.root = options.options.vaultRoot;
|
|
304
|
+
}
|
|
305
|
+
if (options.options.vaultRetention !== undefined) {
|
|
306
|
+
vaultOptionsForStore.retention = options.options.vaultRetention;
|
|
307
|
+
}
|
|
308
|
+
const storeVault = createVault(vaultOptionsForStore);
|
|
309
|
+
let record;
|
|
310
|
+
try {
|
|
311
|
+
record = await storeTextOutput(storeVault, {
|
|
312
|
+
sessionId: options.options.sessionId,
|
|
313
|
+
raw: execution.result.stdout,
|
|
314
|
+
sourceKind: "transform",
|
|
315
|
+
producer,
|
|
316
|
+
lineage,
|
|
317
|
+
decisionIds: [
|
|
318
|
+
decisionId("script-transform-store", options.input.operation.language, lineage.operationHash ?? ""),
|
|
319
|
+
],
|
|
320
|
+
});
|
|
321
|
+
} catch (error) {
|
|
322
|
+
return storageFailure({
|
|
323
|
+
message: `Script-transformed output could not be persisted: ${errorMessage(error)}`,
|
|
324
|
+
preserve: options.preserve,
|
|
325
|
+
lineage,
|
|
326
|
+
});
|
|
327
|
+
}
|
|
328
|
+
const thresholds = { ...DEFAULT_ROUTER_THRESHOLDS, ...options.options.thresholds };
|
|
329
|
+
const primarySource = primaryScriptSourceRef(resolved.sources, options.input.sources);
|
|
330
|
+
const routed = routeTransformedText({
|
|
331
|
+
outputId: record.outputId,
|
|
332
|
+
text: execution.result.stdout,
|
|
333
|
+
preserve: options.preserve,
|
|
334
|
+
thresholds,
|
|
335
|
+
source: primarySource,
|
|
336
|
+
operationKind: "script",
|
|
337
|
+
});
|
|
338
|
+
return {
|
|
339
|
+
toolStatus: "ok",
|
|
340
|
+
decisionId: decisionId("script-transform", record.outputId, options.input.operation.language, routed.routingStatus),
|
|
341
|
+
outputId: record.outputId,
|
|
342
|
+
recordId: record.recordId,
|
|
343
|
+
preserve: options.preserve,
|
|
344
|
+
source: primarySource,
|
|
345
|
+
operation: operationSummary(options.input.operation),
|
|
346
|
+
producer: record.producer,
|
|
347
|
+
persistence: record.persistence,
|
|
348
|
+
lineage,
|
|
349
|
+
routing: {
|
|
350
|
+
status: routed.routingStatus,
|
|
351
|
+
route: "transform",
|
|
352
|
+
reason: routed.reason,
|
|
353
|
+
},
|
|
354
|
+
summary: `Script transform ${options.input.operation.language} completed with ${byteLength(execution.result.stdout)} stdout bytes and ${byteLength(execution.result.stderr)} stderr bytes.`,
|
|
355
|
+
evidence: routed.evidence,
|
|
356
|
+
recovery: {
|
|
357
|
+
how: `Use freeflow_search with source.kind=vault and outputId=${record.outputId}, stream=raw, and an exact lineRange to recover exact script-transformed content. Source evidence remains linked in lineage.sourceOutputIds.`,
|
|
358
|
+
outputId: record.outputId,
|
|
359
|
+
},
|
|
360
|
+
};
|
|
346
361
|
}
|
|
347
362
|
function primaryScriptSourceRef(resolvedSources, inputSources) {
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
363
|
+
const resolved = resolvedSources[0];
|
|
364
|
+
if (resolved) {
|
|
365
|
+
return { kind: "vault", outputId: resolved.outputId, stream: resolved.stream };
|
|
366
|
+
}
|
|
367
|
+
const input = inputSources[0];
|
|
368
|
+
if (input?.stream !== undefined) {
|
|
369
|
+
return { kind: "vault", outputId: input.outputId, stream: input.stream };
|
|
370
|
+
}
|
|
371
|
+
return { kind: "vault", outputId: input?.outputId ?? "unknown" };
|
|
357
372
|
}
|
|
358
373
|
export async function executeSandboxedScriptOperation(options) {
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
}
|
|
372
|
-
const adapterSelection = await selectScriptSandboxAdapter(options.operation.language, config, options.scriptSandboxAdapters ?? []);
|
|
373
|
-
if (!adapterSelection.ok) {
|
|
374
|
-
const failure = {
|
|
375
|
-
ok: false,
|
|
376
|
-
failureKind: "adapter_unavailable",
|
|
377
|
-
executionStatus: "unavailable",
|
|
378
|
-
message: `${adapterSelection.status.reason} No script code was executed.`,
|
|
379
|
-
limits,
|
|
380
|
-
operation,
|
|
381
|
-
failedProofs: adapterSelection.status.failedProofs,
|
|
382
|
-
};
|
|
383
|
-
if (adapterSelection.status.adapterId !== undefined) {
|
|
384
|
-
failure.adapterId = adapterSelection.status.adapterId;
|
|
385
|
-
}
|
|
386
|
-
if (adapterSelection.status.adapterVersion !== undefined) {
|
|
387
|
-
failure.adapterVersion = adapterSelection.status.adapterVersion;
|
|
388
|
-
}
|
|
389
|
-
return failure;
|
|
390
|
-
}
|
|
391
|
-
const execution = await runScriptAdapter({
|
|
392
|
-
adapter: adapterSelection.adapter,
|
|
393
|
-
operation: options.operation,
|
|
394
|
-
resolvedSources: [],
|
|
395
|
-
limits,
|
|
396
|
-
config,
|
|
397
|
-
});
|
|
398
|
-
if (!execution.ok) {
|
|
399
|
-
return {
|
|
400
|
-
ok: false,
|
|
401
|
-
failureKind: "transform_execution_failure",
|
|
402
|
-
executionStatus: "failed",
|
|
403
|
-
message: execution.message,
|
|
404
|
-
limits,
|
|
405
|
-
operation,
|
|
406
|
-
adapterId: adapterSelection.adapter.id,
|
|
407
|
-
adapterVersion: adapterSelection.adapter.version,
|
|
408
|
-
};
|
|
409
|
-
}
|
|
410
|
-
const success = {
|
|
411
|
-
ok: true,
|
|
412
|
-
result: execution.result,
|
|
413
|
-
limits,
|
|
414
|
-
operation,
|
|
415
|
-
adapterId: adapterSelection.adapter.id,
|
|
416
|
-
adapterVersion: adapterSelection.adapter.version,
|
|
374
|
+
const config = effectiveScriptTransformConfig(options.scriptTransform);
|
|
375
|
+
const limits = effectiveScriptLimits(config, options.limits);
|
|
376
|
+
const operation = scriptOperationSummary(options.operation);
|
|
377
|
+
if (!config.enabled) {
|
|
378
|
+
return {
|
|
379
|
+
ok: false,
|
|
380
|
+
failureKind: "script_transform_disabled",
|
|
381
|
+
executionStatus: "unavailable",
|
|
382
|
+
message:
|
|
383
|
+
"Script transform is disabled by default. Enable scriptTransform.enabled only after a sandbox adapter has passed capability probes and review. No script code was executed.",
|
|
384
|
+
limits,
|
|
385
|
+
operation,
|
|
417
386
|
};
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
387
|
+
}
|
|
388
|
+
const adapterSelection = await selectScriptSandboxAdapter(
|
|
389
|
+
options.operation.language,
|
|
390
|
+
config,
|
|
391
|
+
options.scriptSandboxAdapters ?? [],
|
|
392
|
+
);
|
|
393
|
+
if (!adapterSelection.ok) {
|
|
394
|
+
const failure = {
|
|
395
|
+
ok: false,
|
|
396
|
+
failureKind: "adapter_unavailable",
|
|
397
|
+
executionStatus: "unavailable",
|
|
398
|
+
message: `${adapterSelection.status.reason} No script code was executed.`,
|
|
399
|
+
limits,
|
|
400
|
+
operation,
|
|
401
|
+
failedProofs: adapterSelection.status.failedProofs,
|
|
402
|
+
};
|
|
403
|
+
if (adapterSelection.status.adapterId !== undefined) {
|
|
404
|
+
failure.adapterId = adapterSelection.status.adapterId;
|
|
405
|
+
}
|
|
406
|
+
if (adapterSelection.status.adapterVersion !== undefined) {
|
|
407
|
+
failure.adapterVersion = adapterSelection.status.adapterVersion;
|
|
408
|
+
}
|
|
409
|
+
return failure;
|
|
410
|
+
}
|
|
411
|
+
const execution = await runScriptAdapter({
|
|
412
|
+
adapter: adapterSelection.adapter,
|
|
413
|
+
operation: options.operation,
|
|
414
|
+
resolvedSources: [],
|
|
415
|
+
limits,
|
|
416
|
+
config,
|
|
417
|
+
});
|
|
418
|
+
if (!execution.ok) {
|
|
419
|
+
return {
|
|
420
|
+
ok: false,
|
|
421
|
+
failureKind: "transform_execution_failure",
|
|
422
|
+
executionStatus: "failed",
|
|
423
|
+
message: execution.message,
|
|
424
|
+
limits,
|
|
425
|
+
operation,
|
|
426
|
+
adapterId: adapterSelection.adapter.id,
|
|
427
|
+
adapterVersion: adapterSelection.adapter.version,
|
|
428
|
+
};
|
|
429
|
+
}
|
|
430
|
+
const success = {
|
|
431
|
+
ok: true,
|
|
432
|
+
result: execution.result,
|
|
433
|
+
limits,
|
|
434
|
+
operation,
|
|
435
|
+
adapterId: adapterSelection.adapter.id,
|
|
436
|
+
adapterVersion: adapterSelection.adapter.version,
|
|
437
|
+
};
|
|
438
|
+
if (adapterSelection.status.runtime !== undefined) {
|
|
439
|
+
success.runtime = adapterSelection.status.runtime;
|
|
440
|
+
}
|
|
441
|
+
return success;
|
|
422
442
|
}
|
|
423
443
|
async function executeScriptWithAdapter(options) {
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
444
|
+
const execution = await runScriptAdapter(options);
|
|
445
|
+
if (!execution.ok) {
|
|
446
|
+
return execution;
|
|
447
|
+
}
|
|
448
|
+
const stdoutBytes = byteLength(execution.result.stdout ?? "");
|
|
449
|
+
const stderrBytes = byteLength(execution.result.stderr ?? "");
|
|
450
|
+
if (execution.result.status !== "success") {
|
|
451
|
+
const detail = execution.result.reason ? ` ${execution.result.reason}` : "";
|
|
452
|
+
return {
|
|
453
|
+
ok: false,
|
|
454
|
+
message: `Script transform ${options.operation.language} ${execution.result.status}.${detail} stdoutBytes=${stdoutBytes} stderrBytes=${stderrBytes}.`,
|
|
455
|
+
};
|
|
456
|
+
}
|
|
457
|
+
return { ok: true, result: execution.result };
|
|
435
458
|
}
|
|
436
459
|
async function runScriptAdapter(options) {
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
460
|
+
const tempRoot = await mkdtemp(join(tmpdir(), "freeflow-script-transform-"));
|
|
461
|
+
const inputDir = join(tempRoot, "input");
|
|
462
|
+
const workDir = join(tempRoot, "work");
|
|
463
|
+
const outputDir = join(tempRoot, "output");
|
|
464
|
+
try {
|
|
465
|
+
await mkdir(inputDir, { recursive: true });
|
|
466
|
+
await mkdir(workDir, { recursive: true });
|
|
467
|
+
await mkdir(outputDir, { recursive: true });
|
|
468
|
+
const mounts = [];
|
|
469
|
+
for (const source of options.resolvedSources) {
|
|
470
|
+
const path = join(inputDir, `${source.alias}.txt`);
|
|
471
|
+
await writeFile(path, source.text, "utf8");
|
|
472
|
+
mounts.push({ alias: source.alias, path, bytes: source.bytes, sha256: source.textSha256 });
|
|
473
|
+
}
|
|
474
|
+
await writeFile(
|
|
475
|
+
join(inputDir, "manifest.json"),
|
|
476
|
+
JSON.stringify(
|
|
477
|
+
{
|
|
478
|
+
schemaVersion: 1,
|
|
479
|
+
sources: options.resolvedSources.map((source) => ({
|
|
480
|
+
alias: source.alias,
|
|
481
|
+
outputId: source.outputId,
|
|
482
|
+
stream: source.stream,
|
|
483
|
+
bytes: source.bytes,
|
|
484
|
+
sha256: source.textSha256,
|
|
485
|
+
})),
|
|
486
|
+
},
|
|
487
|
+
null,
|
|
488
|
+
2,
|
|
489
|
+
),
|
|
490
|
+
"utf8",
|
|
491
|
+
);
|
|
492
|
+
let result;
|
|
441
493
|
try {
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
limits: options.limits,
|
|
471
|
-
network: options.config.network,
|
|
472
|
-
});
|
|
473
|
-
}
|
|
474
|
-
catch (error) {
|
|
475
|
-
return { ok: false, message: `Script transform adapter ${options.adapter.id} threw before returning a result: ${errorMessage(error)}` };
|
|
476
|
-
}
|
|
477
|
-
const stdoutBytes = byteLength(result.stdout ?? "");
|
|
478
|
-
const stderrBytes = byteLength(result.stderr ?? "");
|
|
479
|
-
if (stdoutBytes + stderrBytes > options.limits.maxOutputBytes) {
|
|
480
|
-
return { ok: false, message: `Script transform output bytes ${stdoutBytes + stderrBytes} exceed maxOutputBytes ${options.limits.maxOutputBytes}.` };
|
|
481
|
-
}
|
|
482
|
-
return { ok: true, result };
|
|
483
|
-
}
|
|
484
|
-
finally {
|
|
485
|
-
await rm(tempRoot, { recursive: true, force: true });
|
|
486
|
-
}
|
|
494
|
+
result = await options.adapter.execute({
|
|
495
|
+
language: options.operation.language,
|
|
496
|
+
code: options.operation.code,
|
|
497
|
+
inputDir,
|
|
498
|
+
workDir,
|
|
499
|
+
outputDir,
|
|
500
|
+
sources: mounts,
|
|
501
|
+
limits: options.limits,
|
|
502
|
+
network: options.config.network,
|
|
503
|
+
});
|
|
504
|
+
} catch (error) {
|
|
505
|
+
return {
|
|
506
|
+
ok: false,
|
|
507
|
+
message: `Script transform adapter ${options.adapter.id} threw before returning a result: ${errorMessage(error)}`,
|
|
508
|
+
};
|
|
509
|
+
}
|
|
510
|
+
const stdoutBytes = byteLength(result.stdout ?? "");
|
|
511
|
+
const stderrBytes = byteLength(result.stderr ?? "");
|
|
512
|
+
if (stdoutBytes + stderrBytes > options.limits.maxOutputBytes) {
|
|
513
|
+
return {
|
|
514
|
+
ok: false,
|
|
515
|
+
message: `Script transform output bytes ${stdoutBytes + stderrBytes} exceed maxOutputBytes ${options.limits.maxOutputBytes}.`,
|
|
516
|
+
};
|
|
517
|
+
}
|
|
518
|
+
return { ok: true, result };
|
|
519
|
+
} finally {
|
|
520
|
+
await rm(tempRoot, { recursive: true, force: true });
|
|
521
|
+
}
|
|
487
522
|
}
|
|
488
523
|
async function resolveScriptSources(options) {
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
}
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
}
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
}
|
|
558
|
-
|
|
524
|
+
const resolved = [];
|
|
525
|
+
let totalBytes = 0;
|
|
526
|
+
for (const source of options.sources) {
|
|
527
|
+
let record;
|
|
528
|
+
try {
|
|
529
|
+
record = await readVaultRecord(options.vault, options.sessionId, source.outputId);
|
|
530
|
+
} catch (error) {
|
|
531
|
+
return {
|
|
532
|
+
ok: false,
|
|
533
|
+
failure: transformSourceUnavailableFailure({
|
|
534
|
+
message: `Script transform source alias=${source.alias} outputId=${source.outputId} could not be found or read: ${errorMessage(error)}`,
|
|
535
|
+
preserve: options.preserve,
|
|
536
|
+
lineage: lineageForScriptInput(
|
|
537
|
+
{ sources: [...options.sources], operation: options.operation },
|
|
538
|
+
{ maxInputBytes: options.maxInputBytes },
|
|
539
|
+
),
|
|
540
|
+
decisionSeed: "script-source-record",
|
|
541
|
+
}),
|
|
542
|
+
};
|
|
543
|
+
}
|
|
544
|
+
const streamResult = resolveSourceStream(record, source.stream);
|
|
545
|
+
if (!streamResult.ok) {
|
|
546
|
+
return {
|
|
547
|
+
ok: false,
|
|
548
|
+
failure: transformValidationFailure({
|
|
549
|
+
message: `Script transform source alias=${source.alias} outputId=${source.outputId} stream is invalid: ${streamResult.message}`,
|
|
550
|
+
preserve: options.preserve,
|
|
551
|
+
lineage: lineageForSource(record, options.operation),
|
|
552
|
+
decisionSeed: "script-source-stream",
|
|
553
|
+
}),
|
|
554
|
+
};
|
|
555
|
+
}
|
|
556
|
+
let text;
|
|
557
|
+
try {
|
|
558
|
+
text = await readOutputText(options.vault, options.sessionId, source.outputId, streamResult.stream);
|
|
559
|
+
} catch (error) {
|
|
560
|
+
return {
|
|
561
|
+
ok: false,
|
|
562
|
+
failure: transformSourceUnavailableFailure({
|
|
563
|
+
message: `Script transform source alias=${source.alias} outputId=${source.outputId} stream=${streamResult.stream} could not be read: ${errorMessage(error)}`,
|
|
564
|
+
preserve: options.preserve,
|
|
565
|
+
lineage: lineageForSource(record, options.operation),
|
|
566
|
+
decisionSeed: "script-source-text",
|
|
567
|
+
}),
|
|
568
|
+
};
|
|
569
|
+
}
|
|
570
|
+
const bytes = byteLength(text);
|
|
571
|
+
totalBytes += bytes;
|
|
572
|
+
if (totalBytes > options.maxInputBytes) {
|
|
573
|
+
return {
|
|
574
|
+
ok: false,
|
|
575
|
+
failure: transformValidationFailure({
|
|
576
|
+
message: `Script transform input bytes ${totalBytes} exceed maxInputBytes ${options.maxInputBytes}.`,
|
|
577
|
+
preserve: options.preserve,
|
|
578
|
+
lineage: lineageForSource(record, options.operation),
|
|
579
|
+
decisionSeed: "script-source-size",
|
|
580
|
+
}),
|
|
581
|
+
};
|
|
582
|
+
}
|
|
583
|
+
resolved.push({
|
|
584
|
+
alias: source.alias,
|
|
585
|
+
outputId: source.outputId,
|
|
586
|
+
recordId: record.recordId,
|
|
587
|
+
stream: streamResult.stream,
|
|
588
|
+
bytes,
|
|
589
|
+
contentHashSha256: record.contentHashSha256,
|
|
590
|
+
textSha256: hashText(text),
|
|
591
|
+
text,
|
|
592
|
+
});
|
|
593
|
+
}
|
|
594
|
+
return { ok: true, sources: resolved };
|
|
559
595
|
}
|
|
560
596
|
function validateTransformSource(value, path, issues) {
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
597
|
+
if (!isRecord(value)) {
|
|
598
|
+
issues.push({ path, message: "Expected transform source object." });
|
|
599
|
+
return;
|
|
600
|
+
}
|
|
601
|
+
if (value.kind !== "vault") {
|
|
602
|
+
issues.push({ path: `${path}.kind`, message: "Slice 5A supports only vault transform sources." });
|
|
603
|
+
return;
|
|
604
|
+
}
|
|
605
|
+
if (typeof value.outputId !== "string" || value.outputId.length === 0) {
|
|
606
|
+
issues.push({ path: `${path}.outputId`, message: "Expected non-empty vault outputId." });
|
|
607
|
+
}
|
|
608
|
+
if (value.stream !== undefined && !isOutputStream(value.stream)) {
|
|
609
|
+
issues.push({ path: `${path}.stream`, message: "Expected a known output stream." });
|
|
610
|
+
}
|
|
575
611
|
}
|
|
576
612
|
function validateTransformOperation(value, path, issues) {
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
613
|
+
if (!isRecord(value)) {
|
|
614
|
+
issues.push({ path, message: "Expected transform operation object." });
|
|
615
|
+
return;
|
|
616
|
+
}
|
|
617
|
+
if (typeof value.kind !== "string" || !OPERATION_KINDS.has(value.kind)) {
|
|
618
|
+
issues.push({ path: `${path}.kind`, message: "Expected a supported transform operation kind." });
|
|
619
|
+
return;
|
|
620
|
+
}
|
|
621
|
+
if (value.kind === "script") {
|
|
622
|
+
validateScriptOperation(value, path, issues);
|
|
623
|
+
return;
|
|
624
|
+
}
|
|
625
|
+
if (value.kind === "jsonExtract") {
|
|
626
|
+
validateJsonExtractOperation(value, path, issues);
|
|
627
|
+
return;
|
|
628
|
+
}
|
|
629
|
+
if (value.kind === "dedupe") {
|
|
630
|
+
validateDedupeOperation(value, path, issues);
|
|
631
|
+
return;
|
|
632
|
+
}
|
|
633
|
+
if (value.kind === "topN") {
|
|
634
|
+
validateTopNOperation(value, path, issues);
|
|
635
|
+
return;
|
|
636
|
+
}
|
|
637
|
+
if (value.kind === "extractUrls" || value.kind === "extractCitations") {
|
|
638
|
+
validateExtractOperation(value, path, issues);
|
|
639
|
+
return;
|
|
640
|
+
}
|
|
641
|
+
if (value.kind === "lineStats" || value.kind === "sizeStats") {
|
|
642
|
+
validateStatsOperation(value, path, issues);
|
|
643
|
+
return;
|
|
644
|
+
}
|
|
645
|
+
if (typeof value.pattern !== "string" || value.pattern.length === 0) {
|
|
646
|
+
issues.push({ path: `${path}.pattern`, message: "Expected a non-empty regex pattern string." });
|
|
647
|
+
}
|
|
648
|
+
validateRegexFlags(value.flags, `${path}.flags`, issues);
|
|
649
|
+
if (value.kind === "regexFilter") {
|
|
650
|
+
if (value.contextLines !== undefined) {
|
|
651
|
+
validateIntegerRange(value.contextLines, `${path}.contextLines`, 0, MAX_CONTEXT_LINES, issues);
|
|
600
652
|
}
|
|
601
|
-
if (value.
|
|
602
|
-
|
|
603
|
-
return;
|
|
604
|
-
}
|
|
605
|
-
if (value.kind === "lineStats" || value.kind === "sizeStats") {
|
|
606
|
-
validateStatsOperation(value, path, issues);
|
|
607
|
-
return;
|
|
608
|
-
}
|
|
609
|
-
if (typeof value.pattern !== "string" || value.pattern.length === 0) {
|
|
610
|
-
issues.push({ path: `${path}.pattern`, message: "Expected a non-empty regex pattern string." });
|
|
653
|
+
if (value.maxMatches !== undefined) {
|
|
654
|
+
validateIntegerRange(value.maxMatches, `${path}.maxMatches`, 1, MAX_MATCHES, issues);
|
|
611
655
|
}
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
validateIntegerRange(value.maxMatches, `${path}.maxMatches`, 1, MAX_MATCHES, issues);
|
|
619
|
-
}
|
|
620
|
-
return;
|
|
656
|
+
return;
|
|
657
|
+
}
|
|
658
|
+
if (value.kind === "groupByRegex") {
|
|
659
|
+
validateGroupSelector(value.group, `${path}.group`, issues);
|
|
660
|
+
if (value.maxGroups !== undefined) {
|
|
661
|
+
validateIntegerRange(value.maxGroups, `${path}.maxGroups`, 1, MAX_GROUPS, issues);
|
|
621
662
|
}
|
|
622
|
-
if (value.
|
|
623
|
-
|
|
624
|
-
if (value.maxGroups !== undefined) {
|
|
625
|
-
validateIntegerRange(value.maxGroups, `${path}.maxGroups`, 1, MAX_GROUPS, issues);
|
|
626
|
-
}
|
|
627
|
-
if (value.maxLinesPerGroup !== undefined) {
|
|
628
|
-
validateIntegerRange(value.maxLinesPerGroup, `${path}.maxLinesPerGroup`, 1, MAX_LINES_PER_GROUP, issues);
|
|
629
|
-
}
|
|
630
|
-
return;
|
|
663
|
+
if (value.maxLinesPerGroup !== undefined) {
|
|
664
|
+
validateIntegerRange(value.maxLinesPerGroup, `${path}.maxLinesPerGroup`, 1, MAX_LINES_PER_GROUP, issues);
|
|
631
665
|
}
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
666
|
+
return;
|
|
667
|
+
}
|
|
668
|
+
for (const key of ["contextLines", "maxMatches", "group", "maxGroups", "maxLinesPerGroup"]) {
|
|
669
|
+
if (value[key] !== undefined) {
|
|
670
|
+
issues.push({ path: `${path}.${key}`, message: `Operation ${value.kind} does not accept ${key}.` });
|
|
636
671
|
}
|
|
672
|
+
}
|
|
637
673
|
}
|
|
638
674
|
function validateScriptOperation(value, path, issues) {
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
675
|
+
if (!isStringIn(value.language, SCRIPT_TRANSFORM_LANGUAGES)) {
|
|
676
|
+
issues.push({
|
|
677
|
+
path: `${path}.language`,
|
|
678
|
+
message: `Expected script language ${SCRIPT_TRANSFORM_LANGUAGES.join(", ")}.`,
|
|
679
|
+
});
|
|
680
|
+
}
|
|
681
|
+
if (typeof value.code !== "string" || value.code.length === 0) {
|
|
682
|
+
issues.push({ path: `${path}.code`, message: "Expected non-empty script code string." });
|
|
683
|
+
} else if (byteLength(value.code) > MAX_SCRIPT_CODE_BYTES) {
|
|
684
|
+
issues.push({ path: `${path}.code`, message: `Script code must be at most ${MAX_SCRIPT_CODE_BYTES} bytes.` });
|
|
685
|
+
}
|
|
686
|
+
if (value.label !== undefined && (typeof value.label !== "string" || value.label.length === 0)) {
|
|
687
|
+
issues.push({ path: `${path}.label`, message: "Expected non-empty script label string when present." });
|
|
688
|
+
}
|
|
651
689
|
}
|
|
652
690
|
function validateScriptTransformSources(value, path, issues) {
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
691
|
+
if (!Array.isArray(value) || value.length === 0) {
|
|
692
|
+
issues.push({ path, message: "Expected one or more script transform vault sources." });
|
|
693
|
+
return;
|
|
694
|
+
}
|
|
695
|
+
const aliases = new Set();
|
|
696
|
+
value.forEach((source, index) => {
|
|
697
|
+
const sourcePath = `${path}[${index}]`;
|
|
698
|
+
validateTransformSource(source, sourcePath, issues);
|
|
699
|
+
if (!isRecord(source)) {
|
|
700
|
+
return;
|
|
701
|
+
}
|
|
702
|
+
if (typeof source.alias !== "string" || !SCRIPT_ALIAS_PATTERN.test(source.alias)) {
|
|
703
|
+
issues.push({ path: `${sourcePath}.alias`, message: "Expected alias matching ^[A-Za-z][A-Za-z0-9_-]{0,63}$." });
|
|
704
|
+
return;
|
|
705
|
+
}
|
|
706
|
+
if (aliases.has(source.alias)) {
|
|
707
|
+
issues.push({ path: `${sourcePath}.alias`, message: `Duplicate script source alias ${source.alias}.` });
|
|
708
|
+
}
|
|
709
|
+
aliases.add(source.alias);
|
|
710
|
+
});
|
|
673
711
|
}
|
|
674
712
|
function validateScriptTransformLimits(value, path, issues) {
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
713
|
+
if (value === undefined) {
|
|
714
|
+
return;
|
|
715
|
+
}
|
|
716
|
+
if (!isRecord(value)) {
|
|
717
|
+
issues.push({ path, message: "Expected script transform limits object." });
|
|
718
|
+
return;
|
|
719
|
+
}
|
|
720
|
+
validateOptionalIntegerRange(value.timeoutMs, `${path}.timeoutMs`, 1, MAX_SCRIPT_TRANSFORM_LIMITS.timeoutMs, issues);
|
|
721
|
+
validateOptionalIntegerRange(
|
|
722
|
+
value.maxInputBytes,
|
|
723
|
+
`${path}.maxInputBytes`,
|
|
724
|
+
1,
|
|
725
|
+
MAX_SCRIPT_TRANSFORM_LIMITS.maxInputBytes,
|
|
726
|
+
issues,
|
|
727
|
+
);
|
|
728
|
+
validateOptionalIntegerRange(
|
|
729
|
+
value.maxOutputBytes,
|
|
730
|
+
`${path}.maxOutputBytes`,
|
|
731
|
+
1,
|
|
732
|
+
MAX_SCRIPT_TRANSFORM_LIMITS.maxOutputBytes,
|
|
733
|
+
issues,
|
|
734
|
+
);
|
|
685
735
|
}
|
|
686
736
|
function validateOptionalIntegerRange(value, path, min, max, issues) {
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
737
|
+
if (value === undefined) {
|
|
738
|
+
return;
|
|
739
|
+
}
|
|
740
|
+
validateIntegerRange(value, path, min, max, issues);
|
|
691
741
|
}
|
|
692
742
|
function validateJsonExtractOperation(value, path, issues) {
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
if (value.path !==
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
}
|
|
724
|
-
}
|
|
743
|
+
const hasPointer = value.pointer !== undefined;
|
|
744
|
+
const hasPath = value.path !== undefined;
|
|
745
|
+
if (hasPointer === hasPath) {
|
|
746
|
+
issues.push({ path, message: "Expected exactly one JSON selector: pointer or path." });
|
|
747
|
+
}
|
|
748
|
+
if (value.pointer !== undefined) {
|
|
749
|
+
if (typeof value.pointer !== "string") {
|
|
750
|
+
issues.push({ path: `${path}.pointer`, message: "Expected JSON pointer string." });
|
|
751
|
+
} else {
|
|
752
|
+
const pointer = parseJsonPointer(value.pointer);
|
|
753
|
+
if (!pointer.ok) {
|
|
754
|
+
issues.push({ path: `${path}.pointer`, message: `Invalid JSON pointer: ${pointer.message}` });
|
|
755
|
+
}
|
|
756
|
+
}
|
|
757
|
+
}
|
|
758
|
+
if (value.path !== undefined) {
|
|
759
|
+
if (typeof value.path !== "string") {
|
|
760
|
+
issues.push({ path: `${path}.path`, message: "Expected JSON path string." });
|
|
761
|
+
} else {
|
|
762
|
+
const parsedPath = parseJsonPath(value.path);
|
|
763
|
+
if (!parsedPath.ok) {
|
|
764
|
+
issues.push({ path: `${path}.path`, message: `Invalid JSON path: ${parsedPath.message}` });
|
|
765
|
+
}
|
|
766
|
+
}
|
|
767
|
+
}
|
|
768
|
+
for (const key of ["pattern", "flags", "contextLines", "maxMatches"]) {
|
|
769
|
+
if (value[key] !== undefined) {
|
|
770
|
+
issues.push({ path: `${path}.${key}`, message: `Operation jsonExtract does not accept ${key}.` });
|
|
771
|
+
}
|
|
772
|
+
}
|
|
725
773
|
}
|
|
726
774
|
function validateDedupeOperation(value, path, issues) {
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
775
|
+
if (value.trim !== undefined && typeof value.trim !== "boolean") {
|
|
776
|
+
issues.push({ path: `${path}.trim`, message: "Expected boolean when present." });
|
|
777
|
+
}
|
|
778
|
+
if (value.caseSensitive !== undefined && typeof value.caseSensitive !== "boolean") {
|
|
779
|
+
issues.push({ path: `${path}.caseSensitive`, message: "Expected boolean when present." });
|
|
780
|
+
}
|
|
781
|
+
if (value.maxLines !== undefined) {
|
|
782
|
+
validateIntegerRange(value.maxLines, `${path}.maxLines`, 1, MAX_DEDUPE_LINES, issues);
|
|
783
|
+
}
|
|
784
|
+
for (const key of [
|
|
785
|
+
"pattern",
|
|
786
|
+
"flags",
|
|
787
|
+
"contextLines",
|
|
788
|
+
"maxMatches",
|
|
789
|
+
"group",
|
|
790
|
+
"maxGroups",
|
|
791
|
+
"maxLinesPerGroup",
|
|
792
|
+
"limit",
|
|
793
|
+
"sort",
|
|
794
|
+
"order",
|
|
795
|
+
]) {
|
|
796
|
+
if (value[key] !== undefined) {
|
|
797
|
+
issues.push({ path: `${path}.${key}`, message: `Operation dedupe does not accept ${key}.` });
|
|
798
|
+
}
|
|
799
|
+
}
|
|
741
800
|
}
|
|
742
801
|
function validateTopNOperation(value, path, issues) {
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
}
|
|
749
|
-
validateRegexFlags(value.flags, `${path}.flags`, issues);
|
|
750
|
-
validateGroupSelector(value.group, `${path}.group`, issues);
|
|
751
|
-
}
|
|
752
|
-
else {
|
|
753
|
-
if (value.flags !== undefined) {
|
|
754
|
-
issues.push({ path: `${path}.flags`, message: "topN flags require a pattern." });
|
|
755
|
-
}
|
|
756
|
-
if (value.group !== undefined) {
|
|
757
|
-
issues.push({ path: `${path}.group`, message: "topN group requires a pattern." });
|
|
758
|
-
}
|
|
759
|
-
}
|
|
760
|
-
if (value.sort !== undefined && value.sort !== "text" && value.sort !== "numeric") {
|
|
761
|
-
issues.push({ path: `${path}.sort`, message: "Expected sort text or numeric." });
|
|
762
|
-
}
|
|
763
|
-
if (value.order !== undefined && value.order !== "asc" && value.order !== "desc") {
|
|
764
|
-
issues.push({ path: `${path}.order`, message: "Expected order asc or desc." });
|
|
765
|
-
}
|
|
766
|
-
for (const key of ["contextLines", "maxMatches", "maxGroups", "maxLinesPerGroup", "trim", "caseSensitive", "maxLines"]) {
|
|
767
|
-
if (value[key] !== undefined) {
|
|
768
|
-
issues.push({ path: `${path}.${key}`, message: `Operation topN does not accept ${key}.` });
|
|
769
|
-
}
|
|
802
|
+
validateIntegerRange(value.limit, `${path}.limit`, 1, MAX_TOP_N_LIMIT, issues);
|
|
803
|
+
const hasPattern = value.pattern !== undefined;
|
|
804
|
+
if (hasPattern) {
|
|
805
|
+
if (typeof value.pattern !== "string" || value.pattern.length === 0) {
|
|
806
|
+
issues.push({ path: `${path}.pattern`, message: "Expected a non-empty regex pattern string." });
|
|
770
807
|
}
|
|
808
|
+
validateRegexFlags(value.flags, `${path}.flags`, issues);
|
|
809
|
+
validateGroupSelector(value.group, `${path}.group`, issues);
|
|
810
|
+
} else {
|
|
811
|
+
if (value.flags !== undefined) {
|
|
812
|
+
issues.push({ path: `${path}.flags`, message: "topN flags require a pattern." });
|
|
813
|
+
}
|
|
814
|
+
if (value.group !== undefined) {
|
|
815
|
+
issues.push({ path: `${path}.group`, message: "topN group requires a pattern." });
|
|
816
|
+
}
|
|
817
|
+
}
|
|
818
|
+
if (value.sort !== undefined && value.sort !== "text" && value.sort !== "numeric") {
|
|
819
|
+
issues.push({ path: `${path}.sort`, message: "Expected sort text or numeric." });
|
|
820
|
+
}
|
|
821
|
+
if (value.order !== undefined && value.order !== "asc" && value.order !== "desc") {
|
|
822
|
+
issues.push({ path: `${path}.order`, message: "Expected order asc or desc." });
|
|
823
|
+
}
|
|
824
|
+
for (const key of [
|
|
825
|
+
"contextLines",
|
|
826
|
+
"maxMatches",
|
|
827
|
+
"maxGroups",
|
|
828
|
+
"maxLinesPerGroup",
|
|
829
|
+
"trim",
|
|
830
|
+
"caseSensitive",
|
|
831
|
+
"maxLines",
|
|
832
|
+
]) {
|
|
833
|
+
if (value[key] !== undefined) {
|
|
834
|
+
issues.push({ path: `${path}.${key}`, message: `Operation topN does not accept ${key}.` });
|
|
835
|
+
}
|
|
836
|
+
}
|
|
771
837
|
}
|
|
772
838
|
function validateExtractOperation(value, path, issues) {
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
839
|
+
if (value.maxMatches !== undefined) {
|
|
840
|
+
validateIntegerRange(value.maxMatches, `${path}.maxMatches`, 1, MAX_EXTRACT_MATCHES, issues);
|
|
841
|
+
}
|
|
842
|
+
if (value.kind === "extractUrls" && value.dedupe !== undefined && typeof value.dedupe !== "boolean") {
|
|
843
|
+
issues.push({ path: `${path}.dedupe`, message: "Expected boolean when present." });
|
|
844
|
+
}
|
|
845
|
+
if (value.kind === "extractCitations" && value.dedupe !== undefined) {
|
|
846
|
+
issues.push({ path: `${path}.dedupe`, message: "Operation extractCitations does not accept dedupe." });
|
|
847
|
+
}
|
|
848
|
+
for (const key of [
|
|
849
|
+
"pattern",
|
|
850
|
+
"flags",
|
|
851
|
+
"contextLines",
|
|
852
|
+
"group",
|
|
853
|
+
"maxGroups",
|
|
854
|
+
"maxLinesPerGroup",
|
|
855
|
+
"limit",
|
|
856
|
+
"sort",
|
|
857
|
+
"order",
|
|
858
|
+
"trim",
|
|
859
|
+
"caseSensitive",
|
|
860
|
+
"maxLines",
|
|
861
|
+
]) {
|
|
862
|
+
if (value[key] !== undefined) {
|
|
863
|
+
issues.push({ path: `${path}.${key}`, message: `Operation ${value.kind} does not accept ${key}.` });
|
|
864
|
+
}
|
|
865
|
+
}
|
|
787
866
|
}
|
|
788
867
|
function validateStatsOperation(value, path, issues) {
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
868
|
+
for (const key of [
|
|
869
|
+
"pattern",
|
|
870
|
+
"flags",
|
|
871
|
+
"contextLines",
|
|
872
|
+
"maxMatches",
|
|
873
|
+
"group",
|
|
874
|
+
"maxGroups",
|
|
875
|
+
"maxLinesPerGroup",
|
|
876
|
+
"limit",
|
|
877
|
+
"sort",
|
|
878
|
+
"order",
|
|
879
|
+
"trim",
|
|
880
|
+
"caseSensitive",
|
|
881
|
+
"maxLines",
|
|
882
|
+
"dedupe",
|
|
883
|
+
]) {
|
|
884
|
+
if (value[key] !== undefined) {
|
|
885
|
+
issues.push({ path: `${path}.${key}`, message: `Operation ${value.kind} does not accept ${key}.` });
|
|
793
886
|
}
|
|
887
|
+
}
|
|
794
888
|
}
|
|
795
889
|
function validateGroupSelector(value, path, issues) {
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
890
|
+
if (value === undefined) {
|
|
891
|
+
return;
|
|
892
|
+
}
|
|
893
|
+
if (typeof value === "number") {
|
|
894
|
+
if (!Number.isInteger(value) || value < 0) {
|
|
895
|
+
issues.push({ path, message: "Expected a non-negative integer group index." });
|
|
896
|
+
}
|
|
897
|
+
return;
|
|
898
|
+
}
|
|
899
|
+
if (typeof value === "string" && value.length > 0) {
|
|
900
|
+
return;
|
|
901
|
+
}
|
|
902
|
+
issues.push({ path, message: "Expected a non-empty string group name or non-negative integer group index." });
|
|
809
903
|
}
|
|
810
904
|
function validateRegexFlags(value, path, issues) {
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
905
|
+
if (value === undefined) {
|
|
906
|
+
return;
|
|
907
|
+
}
|
|
908
|
+
if (typeof value !== "string") {
|
|
909
|
+
issues.push({ path, message: "Expected regex flags string when present." });
|
|
910
|
+
return;
|
|
911
|
+
}
|
|
912
|
+
const seen = new Set();
|
|
913
|
+
for (const flag of value) {
|
|
914
|
+
if (!REGEX_FLAGS.has(flag)) {
|
|
915
|
+
issues.push({ path, message: "Expected regex flags to contain only g, i, m, s, or u." });
|
|
916
|
+
return;
|
|
917
|
+
}
|
|
918
|
+
if (seen.has(flag)) {
|
|
919
|
+
issues.push({ path, message: "Expected regex flags without duplicates." });
|
|
920
|
+
return;
|
|
921
|
+
}
|
|
922
|
+
seen.add(flag);
|
|
923
|
+
}
|
|
830
924
|
}
|
|
831
925
|
function validateIntegerRange(value, path, min, max, issues) {
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
926
|
+
if (!Number.isInteger(value) || value < min || value > max) {
|
|
927
|
+
issues.push({ path, message: `Expected integer from ${min} to ${max}.` });
|
|
928
|
+
}
|
|
835
929
|
}
|
|
836
930
|
function prepareTransformOperation(operation) {
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
return
|
|
931
|
+
if (operation.kind === "jsonExtract") {
|
|
932
|
+
const preparedJson = prepareJsonExtractOperation(operation);
|
|
933
|
+
if (!preparedJson.ok) {
|
|
934
|
+
return preparedJson;
|
|
935
|
+
}
|
|
936
|
+
return { ok: true, value: { kind: "json", value: preparedJson.value } };
|
|
937
|
+
}
|
|
938
|
+
if (
|
|
939
|
+
operation.kind === "dedupe" ||
|
|
940
|
+
operation.kind === "extractUrls" ||
|
|
941
|
+
operation.kind === "extractCitations" ||
|
|
942
|
+
operation.kind === "lineStats" ||
|
|
943
|
+
operation.kind === "sizeStats" ||
|
|
944
|
+
(operation.kind === "topN" && operation.pattern === undefined)
|
|
945
|
+
) {
|
|
946
|
+
return { ok: true, value: { kind: "none" } };
|
|
947
|
+
}
|
|
948
|
+
const compiledRegex = compileRegexOperation(operation);
|
|
949
|
+
if (!compiledRegex.ok) {
|
|
950
|
+
return compiledRegex;
|
|
951
|
+
}
|
|
952
|
+
return { ok: true, value: { kind: "regex", value: compiledRegex.value } };
|
|
857
953
|
}
|
|
858
954
|
function compileRegexOperation(operation) {
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
}
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
}
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
}
|
|
955
|
+
const flags = normalizeRegexFlags(operation.flags);
|
|
956
|
+
let regex;
|
|
957
|
+
try {
|
|
958
|
+
regex = new RegExp(operation.pattern, ensureGlobalFlag(flags));
|
|
959
|
+
} catch (error) {
|
|
960
|
+
return { ok: false, message: `Invalid regex pattern for ${operation.kind}: ${errorMessage(error)}` };
|
|
961
|
+
}
|
|
962
|
+
try {
|
|
963
|
+
const zeroWidthCheck = new RegExp(operation.pattern, flags);
|
|
964
|
+
if (zeroWidthCheck.test("")) {
|
|
965
|
+
return {
|
|
966
|
+
ok: false,
|
|
967
|
+
message: `Invalid regex pattern for ${operation.kind}: patterns that match empty strings are not supported.`,
|
|
968
|
+
};
|
|
969
|
+
}
|
|
970
|
+
} catch (error) {
|
|
971
|
+
return { ok: false, message: `Invalid regex pattern for ${operation.kind}: ${errorMessage(error)}` };
|
|
972
|
+
}
|
|
973
|
+
return {
|
|
974
|
+
ok: true,
|
|
975
|
+
value: {
|
|
976
|
+
regex,
|
|
977
|
+
displayPattern: operation.pattern,
|
|
978
|
+
flags,
|
|
979
|
+
},
|
|
980
|
+
};
|
|
884
981
|
}
|
|
885
982
|
function prepareJsonExtractOperation(operation) {
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
}
|
|
891
|
-
return {
|
|
892
|
-
ok: true,
|
|
893
|
-
value: {
|
|
894
|
-
selectorKind: "pointer",
|
|
895
|
-
selector: operation.pointer,
|
|
896
|
-
segments: pointer.segments,
|
|
897
|
-
},
|
|
898
|
-
};
|
|
983
|
+
if (operation.pointer !== undefined) {
|
|
984
|
+
const pointer = parseJsonPointer(operation.pointer);
|
|
985
|
+
if (!pointer.ok) {
|
|
986
|
+
return { ok: false, message: `Invalid JSON pointer: ${pointer.message}` };
|
|
899
987
|
}
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
988
|
+
return {
|
|
989
|
+
ok: true,
|
|
990
|
+
value: {
|
|
991
|
+
selectorKind: "pointer",
|
|
992
|
+
selector: operation.pointer,
|
|
993
|
+
segments: pointer.segments,
|
|
994
|
+
},
|
|
995
|
+
};
|
|
996
|
+
}
|
|
997
|
+
if (operation.path !== undefined) {
|
|
998
|
+
const path = parseJsonPath(operation.path);
|
|
999
|
+
if (!path.ok) {
|
|
1000
|
+
return { ok: false, message: `Invalid JSON path: ${path.message}` };
|
|
913
1001
|
}
|
|
914
|
-
return {
|
|
1002
|
+
return {
|
|
1003
|
+
ok: true,
|
|
1004
|
+
value: {
|
|
1005
|
+
selectorKind: "path",
|
|
1006
|
+
selector: operation.path,
|
|
1007
|
+
segments: path.segments,
|
|
1008
|
+
},
|
|
1009
|
+
};
|
|
1010
|
+
}
|
|
1011
|
+
return { ok: false, message: "jsonExtract requires exactly one JSON selector: pointer or path." };
|
|
915
1012
|
}
|
|
916
1013
|
function transformText(options) {
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
});
|
|
934
|
-
}
|
|
935
|
-
if (options.operation.kind === "topN") {
|
|
936
|
-
const topNOptions = {
|
|
937
|
-
text: options.text,
|
|
938
|
-
sourceLabel: options.sourceLabel,
|
|
939
|
-
operation: options.operation,
|
|
940
|
-
};
|
|
941
|
-
if (options.prepared.kind === "regex") {
|
|
942
|
-
topNOptions.compiled = options.prepared.value;
|
|
943
|
-
}
|
|
944
|
-
return transformTopN(topNOptions);
|
|
945
|
-
}
|
|
946
|
-
if (options.operation.kind === "extractUrls") {
|
|
947
|
-
return transformExtractUrls({ text: options.text, sourceLabel: options.sourceLabel, operation: options.operation });
|
|
948
|
-
}
|
|
949
|
-
if (options.operation.kind === "extractCitations") {
|
|
950
|
-
return transformExtractCitations({ text: options.text, sourceLabel: options.sourceLabel, operation: options.operation });
|
|
951
|
-
}
|
|
952
|
-
if (options.operation.kind === "lineStats") {
|
|
953
|
-
return transformLineStats({ text: options.text, sourceLabel: options.sourceLabel });
|
|
954
|
-
}
|
|
955
|
-
if (options.operation.kind === "sizeStats") {
|
|
956
|
-
return transformSizeStats({ text: options.text, sourceLabel: options.sourceLabel });
|
|
957
|
-
}
|
|
958
|
-
if (options.prepared.kind !== "regex") {
|
|
959
|
-
throw new Error(`${options.operation.kind} operation was not prepared with a regex.`);
|
|
960
|
-
}
|
|
961
|
-
if (options.operation.kind === "regexFilter") {
|
|
962
|
-
return transformRegexFilter({
|
|
963
|
-
text: options.text,
|
|
964
|
-
sourceLabel: options.sourceLabel,
|
|
965
|
-
operation: options.operation,
|
|
966
|
-
compiled: options.prepared.value,
|
|
967
|
-
});
|
|
968
|
-
}
|
|
969
|
-
if (options.operation.kind === "groupByRegex") {
|
|
970
|
-
return transformGroupByRegex({
|
|
971
|
-
text: options.text,
|
|
972
|
-
sourceLabel: options.sourceLabel,
|
|
973
|
-
operation: options.operation,
|
|
974
|
-
compiled: options.prepared.value,
|
|
975
|
-
});
|
|
976
|
-
}
|
|
977
|
-
return transformCountMatches({
|
|
978
|
-
text: options.text,
|
|
979
|
-
sourceLabel: options.sourceLabel,
|
|
980
|
-
operation: options.operation,
|
|
981
|
-
compiled: options.prepared.value,
|
|
1014
|
+
if (options.operation.kind === "jsonExtract") {
|
|
1015
|
+
if (options.prepared.kind !== "json") {
|
|
1016
|
+
throw new Error("jsonExtract operation was not prepared with a JSON selector.");
|
|
1017
|
+
}
|
|
1018
|
+
return transformJsonExtract({
|
|
1019
|
+
text: options.text,
|
|
1020
|
+
sourceLabel: options.sourceLabel,
|
|
1021
|
+
operation: options.operation,
|
|
1022
|
+
prepared: options.prepared.value,
|
|
1023
|
+
});
|
|
1024
|
+
}
|
|
1025
|
+
if (options.operation.kind === "dedupe") {
|
|
1026
|
+
return transformDedupe({
|
|
1027
|
+
text: options.text,
|
|
1028
|
+
sourceLabel: options.sourceLabel,
|
|
1029
|
+
operation: options.operation,
|
|
982
1030
|
});
|
|
1031
|
+
}
|
|
1032
|
+
if (options.operation.kind === "topN") {
|
|
1033
|
+
const topNOptions = {
|
|
1034
|
+
text: options.text,
|
|
1035
|
+
sourceLabel: options.sourceLabel,
|
|
1036
|
+
operation: options.operation,
|
|
1037
|
+
};
|
|
1038
|
+
if (options.prepared.kind === "regex") {
|
|
1039
|
+
topNOptions.compiled = options.prepared.value;
|
|
1040
|
+
}
|
|
1041
|
+
return transformTopN(topNOptions);
|
|
1042
|
+
}
|
|
1043
|
+
if (options.operation.kind === "extractUrls") {
|
|
1044
|
+
return transformExtractUrls({ text: options.text, sourceLabel: options.sourceLabel, operation: options.operation });
|
|
1045
|
+
}
|
|
1046
|
+
if (options.operation.kind === "extractCitations") {
|
|
1047
|
+
return transformExtractCitations({
|
|
1048
|
+
text: options.text,
|
|
1049
|
+
sourceLabel: options.sourceLabel,
|
|
1050
|
+
operation: options.operation,
|
|
1051
|
+
});
|
|
1052
|
+
}
|
|
1053
|
+
if (options.operation.kind === "lineStats") {
|
|
1054
|
+
return transformLineStats({ text: options.text, sourceLabel: options.sourceLabel });
|
|
1055
|
+
}
|
|
1056
|
+
if (options.operation.kind === "sizeStats") {
|
|
1057
|
+
return transformSizeStats({ text: options.text, sourceLabel: options.sourceLabel });
|
|
1058
|
+
}
|
|
1059
|
+
if (options.prepared.kind !== "regex") {
|
|
1060
|
+
throw new Error(`${options.operation.kind} operation was not prepared with a regex.`);
|
|
1061
|
+
}
|
|
1062
|
+
if (options.operation.kind === "regexFilter") {
|
|
1063
|
+
return transformRegexFilter({
|
|
1064
|
+
text: options.text,
|
|
1065
|
+
sourceLabel: options.sourceLabel,
|
|
1066
|
+
operation: options.operation,
|
|
1067
|
+
compiled: options.prepared.value,
|
|
1068
|
+
});
|
|
1069
|
+
}
|
|
1070
|
+
if (options.operation.kind === "groupByRegex") {
|
|
1071
|
+
return transformGroupByRegex({
|
|
1072
|
+
text: options.text,
|
|
1073
|
+
sourceLabel: options.sourceLabel,
|
|
1074
|
+
operation: options.operation,
|
|
1075
|
+
compiled: options.prepared.value,
|
|
1076
|
+
});
|
|
1077
|
+
}
|
|
1078
|
+
return transformCountMatches({
|
|
1079
|
+
text: options.text,
|
|
1080
|
+
sourceLabel: options.sourceLabel,
|
|
1081
|
+
operation: options.operation,
|
|
1082
|
+
compiled: options.prepared.value,
|
|
1083
|
+
});
|
|
983
1084
|
}
|
|
984
1085
|
function transformRegexFilter(options) {
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
}
|
|
1086
|
+
const lines = splitLines(options.text);
|
|
1087
|
+
const contextLines = options.operation.contextLines ?? DEFAULT_CONTEXT_LINES;
|
|
1088
|
+
const maxMatches = options.operation.maxMatches ?? DEFAULT_MAX_MATCHES;
|
|
1089
|
+
const stats = collectMatches(lines, options.compiled.regex, maxMatches);
|
|
1090
|
+
const windows = mergeLineWindows(
|
|
1091
|
+
stats.matchedLineNumbers.map((lineNumber) => ({
|
|
1092
|
+
start: Math.max(1, lineNumber - contextLines),
|
|
1093
|
+
end: Math.min(lines.length, lineNumber + contextLines),
|
|
1094
|
+
})),
|
|
1095
|
+
);
|
|
1096
|
+
const parts = [
|
|
1097
|
+
"# freeflow_search action=transform regexFilter",
|
|
1098
|
+
`source: ${options.sourceLabel}`,
|
|
1099
|
+
`pattern: ${formatPattern(options.compiled)}`,
|
|
1100
|
+
`contextLines: ${contextLines}`,
|
|
1101
|
+
`maxMatches: ${maxMatches}`,
|
|
1102
|
+
`matches: ${stats.matches}`,
|
|
1103
|
+
`matchedLines: ${stats.matchedLines}`,
|
|
1104
|
+
`truncated: ${stats.truncated}`,
|
|
1105
|
+
];
|
|
1106
|
+
for (const window of windows) {
|
|
1107
|
+
parts.push("", `@@ source lines ${window.start}-${window.end} @@`);
|
|
1108
|
+
for (let lineNumber = window.start; lineNumber <= window.end; lineNumber += 1) {
|
|
1109
|
+
parts.push(`${lineNumber}| ${lines[lineNumber - 1] ?? ""}`);
|
|
1110
|
+
}
|
|
1111
|
+
}
|
|
1112
|
+
return {
|
|
1113
|
+
text: `${parts.join("\n")}\n`,
|
|
1114
|
+
summary: `Transformed regexFilter from vaulted ${sourceStreamLabel(options.sourceLabel)} output: ${stats.matches} match(es) across ${stats.matchedLines} line(s).`,
|
|
1115
|
+
stats,
|
|
1116
|
+
};
|
|
1014
1117
|
}
|
|
1015
1118
|
function transformCountMatches(options) {
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1119
|
+
const lines = splitLines(options.text);
|
|
1120
|
+
const stats = collectMatches(lines, options.compiled.regex);
|
|
1121
|
+
const text = [
|
|
1122
|
+
"# freeflow_search action=transform countMatches",
|
|
1123
|
+
`source: ${options.sourceLabel}`,
|
|
1124
|
+
`pattern: ${formatPattern(options.compiled)}`,
|
|
1125
|
+
`matches: ${stats.matches}`,
|
|
1126
|
+
`matchedLines: ${stats.matchedLines}`,
|
|
1127
|
+
"",
|
|
1128
|
+
].join("\n");
|
|
1129
|
+
return {
|
|
1130
|
+
text,
|
|
1131
|
+
summary: `Transformed countMatches from vaulted ${sourceStreamLabel(options.sourceLabel)} output: ${stats.matches} match(es) across ${stats.matchedLines} line(s).`,
|
|
1132
|
+
stats,
|
|
1133
|
+
};
|
|
1031
1134
|
}
|
|
1032
1135
|
function transformGroupByRegex(options) {
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
};
|
|
1136
|
+
const lines = splitLines(options.text);
|
|
1137
|
+
const groupSelector = options.operation.group ?? 1;
|
|
1138
|
+
const maxGroups = options.operation.maxGroups ?? DEFAULT_MAX_GROUPS;
|
|
1139
|
+
const maxLinesPerGroup = options.operation.maxLinesPerGroup ?? DEFAULT_MAX_LINES_PER_GROUP;
|
|
1140
|
+
const groups = new Map();
|
|
1141
|
+
const allGroupKeys = new Set();
|
|
1142
|
+
let matchedLines = 0;
|
|
1143
|
+
let truncated = false;
|
|
1144
|
+
for (let index = 0; index < lines.length; index += 1) {
|
|
1145
|
+
const line = lines[index] ?? "";
|
|
1146
|
+
options.compiled.regex.lastIndex = 0;
|
|
1147
|
+
const match = options.compiled.regex.exec(line);
|
|
1148
|
+
if (!match) {
|
|
1149
|
+
continue;
|
|
1150
|
+
}
|
|
1151
|
+
if (match[0].length === 0) {
|
|
1152
|
+
throw new Error("Regex patterns that produce zero-width matches are not supported.");
|
|
1153
|
+
}
|
|
1154
|
+
matchedLines += 1;
|
|
1155
|
+
const groupKey = matchGroupValue(match, groupSelector, index + 1);
|
|
1156
|
+
allGroupKeys.add(groupKey);
|
|
1157
|
+
let group = groups.get(groupKey);
|
|
1158
|
+
if (!group) {
|
|
1159
|
+
if (groups.size >= maxGroups) {
|
|
1160
|
+
truncated = true;
|
|
1161
|
+
continue;
|
|
1162
|
+
}
|
|
1163
|
+
group = { count: 0, entries: [] };
|
|
1164
|
+
groups.set(groupKey, group);
|
|
1165
|
+
}
|
|
1166
|
+
group.count += 1;
|
|
1167
|
+
if (group.entries.length < maxLinesPerGroup) {
|
|
1168
|
+
group.entries.push({ lineNumber: index + 1, line });
|
|
1169
|
+
} else {
|
|
1170
|
+
truncated = true;
|
|
1171
|
+
}
|
|
1172
|
+
}
|
|
1173
|
+
const parts = [
|
|
1174
|
+
"# freeflow_search action=transform groupByRegex",
|
|
1175
|
+
`source: ${options.sourceLabel}`,
|
|
1176
|
+
`pattern: ${formatPattern(options.compiled)}`,
|
|
1177
|
+
`group: ${String(groupSelector)}`,
|
|
1178
|
+
`maxGroups: ${maxGroups}`,
|
|
1179
|
+
`maxLinesPerGroup: ${maxLinesPerGroup}`,
|
|
1180
|
+
`groups: ${allGroupKeys.size}`,
|
|
1181
|
+
`returnedGroups: ${groups.size}`,
|
|
1182
|
+
`matchedLines: ${matchedLines}`,
|
|
1183
|
+
`truncated: ${truncated}`,
|
|
1184
|
+
];
|
|
1185
|
+
for (const [groupKey, group] of groups.entries()) {
|
|
1186
|
+
parts.push("", `## group: ${groupKey}`, `count: ${group.count}`);
|
|
1187
|
+
for (const entry of group.entries) {
|
|
1188
|
+
parts.push(`${entry.lineNumber}| ${entry.line}`);
|
|
1189
|
+
}
|
|
1190
|
+
}
|
|
1191
|
+
return {
|
|
1192
|
+
text: `${parts.join("\n")}\n`,
|
|
1193
|
+
summary: `Transformed groupByRegex from vaulted ${sourceStreamLabel(options.sourceLabel)} output: ${allGroupKeys.size} group(s), ${matchedLines} matched line(s).`,
|
|
1194
|
+
stats: {
|
|
1195
|
+
matches: matchedLines,
|
|
1196
|
+
matchedLines,
|
|
1197
|
+
matchedLineNumbers: [],
|
|
1198
|
+
truncated,
|
|
1199
|
+
},
|
|
1200
|
+
};
|
|
1099
1201
|
}
|
|
1100
1202
|
function transformDedupe(options) {
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
};
|
|
1203
|
+
const lines = splitLines(options.text);
|
|
1204
|
+
const trim = options.operation.trim ?? false;
|
|
1205
|
+
const caseSensitive = options.operation.caseSensitive ?? true;
|
|
1206
|
+
const maxLines = options.operation.maxLines ?? DEFAULT_MAX_DEDUPE_LINES;
|
|
1207
|
+
const seen = new Set();
|
|
1208
|
+
const uniqueEntries = [];
|
|
1209
|
+
let duplicatesRemoved = 0;
|
|
1210
|
+
let truncated = false;
|
|
1211
|
+
for (let index = 0; index < lines.length; index += 1) {
|
|
1212
|
+
const line = lines[index] ?? "";
|
|
1213
|
+
const displayLine = trim ? line.trim() : line;
|
|
1214
|
+
const key = caseSensitive ? displayLine : displayLine.toLowerCase();
|
|
1215
|
+
if (seen.has(key)) {
|
|
1216
|
+
duplicatesRemoved += 1;
|
|
1217
|
+
continue;
|
|
1218
|
+
}
|
|
1219
|
+
seen.add(key);
|
|
1220
|
+
if (uniqueEntries.length < maxLines) {
|
|
1221
|
+
uniqueEntries.push({ lineNumber: index + 1, line: displayLine });
|
|
1222
|
+
} else {
|
|
1223
|
+
truncated = true;
|
|
1224
|
+
}
|
|
1225
|
+
}
|
|
1226
|
+
const parts = [
|
|
1227
|
+
"# freeflow_search action=transform dedupe",
|
|
1228
|
+
`source: ${options.sourceLabel}`,
|
|
1229
|
+
`trim: ${trim}`,
|
|
1230
|
+
`caseSensitive: ${caseSensitive}`,
|
|
1231
|
+
`maxLines: ${maxLines}`,
|
|
1232
|
+
`inputLines: ${lines.length}`,
|
|
1233
|
+
`uniqueLines: ${seen.size}`,
|
|
1234
|
+
`returnedLines: ${uniqueEntries.length}`,
|
|
1235
|
+
`duplicatesRemoved: ${duplicatesRemoved}`,
|
|
1236
|
+
`truncated: ${truncated}`,
|
|
1237
|
+
"",
|
|
1238
|
+
];
|
|
1239
|
+
for (const entry of uniqueEntries) {
|
|
1240
|
+
parts.push(`${entry.lineNumber}| ${entry.line}`);
|
|
1241
|
+
}
|
|
1242
|
+
return {
|
|
1243
|
+
text: `${parts.join("\n")}\n`,
|
|
1244
|
+
summary: `Transformed dedupe from vaulted ${sourceStreamLabel(options.sourceLabel)} output: ${seen.size} unique line(s), ${duplicatesRemoved} duplicate line(s) removed.`,
|
|
1245
|
+
stats: {
|
|
1246
|
+
matches: seen.size,
|
|
1247
|
+
matchedLines: seen.size,
|
|
1248
|
+
matchedLineNumbers: uniqueEntries.map((entry) => entry.lineNumber),
|
|
1249
|
+
truncated,
|
|
1250
|
+
},
|
|
1251
|
+
};
|
|
1151
1252
|
}
|
|
1152
1253
|
function transformTopN(options) {
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1254
|
+
const lines = splitLines(options.text);
|
|
1255
|
+
const sort = options.operation.sort ?? "text";
|
|
1256
|
+
const order = options.operation.order ?? "asc";
|
|
1257
|
+
const groupSelector = options.operation.group ?? 0;
|
|
1258
|
+
const scored = [];
|
|
1259
|
+
for (let index = 0; index < lines.length; index += 1) {
|
|
1260
|
+
const line = lines[index] ?? "";
|
|
1261
|
+
let scoreText = line;
|
|
1262
|
+
if (options.compiled !== undefined) {
|
|
1263
|
+
options.compiled.regex.lastIndex = 0;
|
|
1264
|
+
const match = options.compiled.regex.exec(line);
|
|
1265
|
+
if (!match) {
|
|
1266
|
+
continue;
|
|
1267
|
+
}
|
|
1268
|
+
if (match[0].length === 0) {
|
|
1269
|
+
throw new Error("Regex patterns that produce zero-width matches are not supported.");
|
|
1270
|
+
}
|
|
1271
|
+
scoreText = matchGroupValue(match, groupSelector, index + 1);
|
|
1272
|
+
}
|
|
1273
|
+
const score = sort === "numeric" ? Number(scoreText) : scoreText;
|
|
1274
|
+
if (sort === "numeric" && (typeof score !== "number" || !Number.isFinite(score))) {
|
|
1275
|
+
throw new Error(`topN numeric score on source line ${index + 1} is not finite: ${scoreText}`);
|
|
1276
|
+
}
|
|
1277
|
+
scored.push({ lineNumber: index + 1, line, score });
|
|
1278
|
+
}
|
|
1279
|
+
const sorted = [...scored].sort((left, right) => compareTopNEntries(left, right, sort, order));
|
|
1280
|
+
const selected = sorted.slice(0, options.operation.limit);
|
|
1281
|
+
const parts = [
|
|
1282
|
+
"# freeflow_search action=transform topN",
|
|
1283
|
+
`source: ${options.sourceLabel}`,
|
|
1284
|
+
...(options.compiled !== undefined
|
|
1285
|
+
? [`pattern: ${formatPattern(options.compiled)}`, `group: ${String(groupSelector)}`]
|
|
1286
|
+
: []),
|
|
1287
|
+
`sort: ${sort}`,
|
|
1288
|
+
`order: ${order}`,
|
|
1289
|
+
`limit: ${options.operation.limit}`,
|
|
1290
|
+
`matchedLines: ${scored.length}`,
|
|
1291
|
+
`returnedLines: ${selected.length}`,
|
|
1292
|
+
`truncated: ${selected.length < scored.length}`,
|
|
1293
|
+
"",
|
|
1294
|
+
];
|
|
1295
|
+
for (const entry of selected) {
|
|
1296
|
+
parts.push(`${entry.lineNumber}| score=${String(entry.score)} | ${entry.line}`);
|
|
1297
|
+
}
|
|
1298
|
+
return {
|
|
1299
|
+
text: `${parts.join("\n")}\n`,
|
|
1300
|
+
summary: `Transformed topN from vaulted ${sourceStreamLabel(options.sourceLabel)} output: returned ${selected.length} of ${scored.length} matched line(s).`,
|
|
1301
|
+
stats: {
|
|
1302
|
+
matches: selected.length,
|
|
1303
|
+
matchedLines: scored.length,
|
|
1304
|
+
matchedLineNumbers: selected.map((entry) => entry.lineNumber),
|
|
1305
|
+
truncated: selected.length < scored.length,
|
|
1306
|
+
},
|
|
1307
|
+
};
|
|
1205
1308
|
}
|
|
1206
1309
|
function transformExtractUrls(options) {
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1310
|
+
const maxMatches = options.operation.maxMatches ?? DEFAULT_MAX_EXTRACT_MATCHES;
|
|
1311
|
+
const dedupe = options.operation.dedupe ?? false;
|
|
1312
|
+
const lines = splitLines(options.text);
|
|
1313
|
+
const seen = new Set();
|
|
1314
|
+
const entries = [];
|
|
1315
|
+
let matches = 0;
|
|
1316
|
+
let truncated = false;
|
|
1317
|
+
lineLoop: for (let index = 0; index < lines.length; index += 1) {
|
|
1318
|
+
const line = lines[index] ?? "";
|
|
1319
|
+
URL_PATTERN.lastIndex = 0;
|
|
1320
|
+
let match;
|
|
1321
|
+
while ((match = URL_PATTERN.exec(line)) !== null) {
|
|
1322
|
+
const rawUrl = match[0];
|
|
1323
|
+
const url = trimUrl(rawUrl);
|
|
1324
|
+
if (url.length === 0) {
|
|
1325
|
+
continue;
|
|
1326
|
+
}
|
|
1327
|
+
matches += 1;
|
|
1328
|
+
if (!dedupe || !seen.has(url)) {
|
|
1329
|
+
seen.add(url);
|
|
1330
|
+
entries.push({ lineNumber: index + 1, url });
|
|
1331
|
+
}
|
|
1332
|
+
if (matches >= maxMatches) {
|
|
1333
|
+
truncated = index < lines.length - 1 || URL_PATTERN.lastIndex < line.length;
|
|
1334
|
+
break lineLoop;
|
|
1335
|
+
}
|
|
1336
|
+
}
|
|
1337
|
+
}
|
|
1338
|
+
const parts = [
|
|
1339
|
+
"# freeflow_search action=transform extractUrls",
|
|
1340
|
+
`source: ${options.sourceLabel}`,
|
|
1341
|
+
`dedupe: ${dedupe}`,
|
|
1342
|
+
`maxMatches: ${maxMatches}`,
|
|
1343
|
+
`matches: ${matches}`,
|
|
1344
|
+
`returnedUrls: ${entries.length}`,
|
|
1345
|
+
`truncated: ${truncated}`,
|
|
1346
|
+
"",
|
|
1347
|
+
];
|
|
1348
|
+
for (const entry of entries) {
|
|
1349
|
+
parts.push(`${entry.lineNumber}| ${entry.url}`);
|
|
1350
|
+
}
|
|
1351
|
+
return {
|
|
1352
|
+
text: `${parts.join("\n")}\n`,
|
|
1353
|
+
summary: `Transformed extractUrls from vaulted ${sourceStreamLabel(options.sourceLabel)} output: ${entries.length} URL(s) returned from ${matches} match(es).`,
|
|
1354
|
+
stats: {
|
|
1355
|
+
matches: entries.length,
|
|
1356
|
+
matchedLines: new Set(entries.map((entry) => entry.lineNumber)).size,
|
|
1357
|
+
matchedLineNumbers: entries.map((entry) => entry.lineNumber),
|
|
1358
|
+
truncated,
|
|
1359
|
+
},
|
|
1360
|
+
};
|
|
1258
1361
|
}
|
|
1259
1362
|
function transformExtractCitations(options) {
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1363
|
+
const maxMatches = options.operation.maxMatches ?? DEFAULT_MAX_EXTRACT_MATCHES;
|
|
1364
|
+
const lines = splitLines(options.text);
|
|
1365
|
+
const entries = [];
|
|
1366
|
+
let truncated = false;
|
|
1367
|
+
lineLoop: for (let index = 0; index < lines.length; index += 1) {
|
|
1368
|
+
const line = lines[index] ?? "";
|
|
1369
|
+
const lineEntries = citationEntriesForLine(line, index + 1);
|
|
1370
|
+
for (let entryIndex = 0; entryIndex < lineEntries.length; entryIndex += 1) {
|
|
1371
|
+
const entry = lineEntries[entryIndex];
|
|
1372
|
+
if (entry === undefined) {
|
|
1373
|
+
continue;
|
|
1374
|
+
}
|
|
1375
|
+
entries.push(entry);
|
|
1376
|
+
if (entries.length >= maxMatches) {
|
|
1377
|
+
truncated = entryIndex < lineEntries.length - 1 || index < lines.length - 1;
|
|
1378
|
+
break lineLoop;
|
|
1379
|
+
}
|
|
1380
|
+
}
|
|
1381
|
+
}
|
|
1382
|
+
const parts = [
|
|
1383
|
+
"# freeflow_search action=transform extractCitations",
|
|
1384
|
+
`source: ${options.sourceLabel}`,
|
|
1385
|
+
`maxMatches: ${maxMatches}`,
|
|
1386
|
+
`citations: ${entries.length}`,
|
|
1387
|
+
`truncated: ${truncated}`,
|
|
1388
|
+
"",
|
|
1389
|
+
];
|
|
1390
|
+
for (const entry of entries) {
|
|
1391
|
+
parts.push(
|
|
1392
|
+
entry.target === undefined
|
|
1393
|
+
? `${entry.lineNumber}| ${entry.type} | ${entry.label}`
|
|
1394
|
+
: `${entry.lineNumber}| ${entry.type} | ${entry.label} | ${entry.target}`,
|
|
1395
|
+
);
|
|
1396
|
+
}
|
|
1397
|
+
return {
|
|
1398
|
+
text: `${parts.join("\n")}\n`,
|
|
1399
|
+
summary: `Transformed extractCitations from vaulted ${sourceStreamLabel(options.sourceLabel)} output: ${entries.length} citation(s) returned.`,
|
|
1400
|
+
stats: {
|
|
1401
|
+
matches: entries.length,
|
|
1402
|
+
matchedLines: new Set(entries.map((entry) => entry.lineNumber)).size,
|
|
1403
|
+
matchedLineNumbers: entries.map((entry) => entry.lineNumber),
|
|
1404
|
+
truncated,
|
|
1405
|
+
},
|
|
1406
|
+
};
|
|
1302
1407
|
}
|
|
1303
1408
|
function transformLineStats(options) {
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
|
|
1409
|
+
const lines = splitLines(options.text);
|
|
1410
|
+
let blankLines = 0;
|
|
1411
|
+
let maxLineBytes = 0;
|
|
1412
|
+
let maxLineNumber = 0;
|
|
1413
|
+
lines.forEach((line, index) => {
|
|
1414
|
+
if (line.trim().length === 0) {
|
|
1415
|
+
blankLines += 1;
|
|
1416
|
+
}
|
|
1417
|
+
const lineBytes = byteLength(line);
|
|
1418
|
+
if (lineBytes > maxLineBytes) {
|
|
1419
|
+
maxLineBytes = lineBytes;
|
|
1420
|
+
maxLineNumber = index + 1;
|
|
1421
|
+
}
|
|
1422
|
+
});
|
|
1423
|
+
const nonEmptyLines = lines.length - blankLines;
|
|
1424
|
+
const text = [
|
|
1425
|
+
"# freeflow_search action=transform lineStats",
|
|
1426
|
+
`source: ${options.sourceLabel}`,
|
|
1427
|
+
`lines: ${lines.length}`,
|
|
1428
|
+
`nonEmptyLines: ${nonEmptyLines}`,
|
|
1429
|
+
`blankLines: ${blankLines}`,
|
|
1430
|
+
`maxLineBytes: ${maxLineBytes}`,
|
|
1431
|
+
`maxLineNumber: ${maxLineNumber}`,
|
|
1432
|
+
"",
|
|
1433
|
+
].join("\n");
|
|
1434
|
+
return {
|
|
1435
|
+
text,
|
|
1436
|
+
summary: `Transformed lineStats from vaulted ${sourceStreamLabel(options.sourceLabel)} output: ${lines.length} line(s), ${nonEmptyLines} non-empty, ${blankLines} blank.`,
|
|
1437
|
+
stats: {
|
|
1438
|
+
matches: lines.length,
|
|
1439
|
+
matchedLines: lines.length,
|
|
1440
|
+
matchedLineNumbers: [],
|
|
1441
|
+
truncated: false,
|
|
1442
|
+
},
|
|
1443
|
+
};
|
|
1339
1444
|
}
|
|
1340
1445
|
function transformSizeStats(options) {
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1446
|
+
const bytes = byteLength(options.text);
|
|
1447
|
+
const utf16CodeUnits = options.text.length;
|
|
1448
|
+
const codePoints = Array.from(options.text).length;
|
|
1449
|
+
const lines = countLines(options.text);
|
|
1450
|
+
const hash = hashText(options.text);
|
|
1451
|
+
const text = [
|
|
1452
|
+
"# freeflow_search action=transform sizeStats",
|
|
1453
|
+
`source: ${options.sourceLabel}`,
|
|
1454
|
+
`bytes: ${bytes}`,
|
|
1455
|
+
`utf16CodeUnits: ${utf16CodeUnits}`,
|
|
1456
|
+
`codePoints: ${codePoints}`,
|
|
1457
|
+
`lines: ${lines}`,
|
|
1458
|
+
`sha256: ${hash}`,
|
|
1459
|
+
"",
|
|
1460
|
+
].join("\n");
|
|
1461
|
+
return {
|
|
1462
|
+
text,
|
|
1463
|
+
summary: `Transformed sizeStats from vaulted ${sourceStreamLabel(options.sourceLabel)} output: ${bytes} byte(s), ${utf16CodeUnits} code unit(s), ${lines} line(s).`,
|
|
1464
|
+
stats: {
|
|
1465
|
+
matches: bytes,
|
|
1466
|
+
matchedLines: lines,
|
|
1467
|
+
matchedLineNumbers: [],
|
|
1468
|
+
truncated: false,
|
|
1469
|
+
},
|
|
1470
|
+
};
|
|
1366
1471
|
}
|
|
1367
1472
|
function transformJsonExtract(options) {
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
};
|
|
1473
|
+
let parsed;
|
|
1474
|
+
try {
|
|
1475
|
+
parsed = JSON.parse(options.text);
|
|
1476
|
+
} catch (error) {
|
|
1477
|
+
throw new Error(`Invalid JSON source for jsonExtract: ${errorMessage(error)}`);
|
|
1478
|
+
}
|
|
1479
|
+
const resolved = resolveJsonSelector(parsed, options.prepared.segments);
|
|
1480
|
+
if (!resolved.ok) {
|
|
1481
|
+
throw new Error(`JSON selector ${options.prepared.selector} did not resolve: ${resolved.message}`);
|
|
1482
|
+
}
|
|
1483
|
+
const valueType = jsonValueType(resolved.value);
|
|
1484
|
+
const valueText = `${JSON.stringify(resolved.value, null, 2)}\n`;
|
|
1485
|
+
const text = [
|
|
1486
|
+
"# freeflow_search action=transform jsonExtract",
|
|
1487
|
+
`source: ${options.sourceLabel}`,
|
|
1488
|
+
`selectorKind: ${options.prepared.selectorKind}`,
|
|
1489
|
+
`selector: ${options.prepared.selector}`,
|
|
1490
|
+
`valueType: ${valueType}`,
|
|
1491
|
+
"",
|
|
1492
|
+
valueText,
|
|
1493
|
+
].join("\n");
|
|
1494
|
+
return {
|
|
1495
|
+
text,
|
|
1496
|
+
summary: `Transformed jsonExtract from vaulted ${sourceStreamLabel(options.sourceLabel)} output using ${options.prepared.selectorKind} ${options.prepared.selector}.`,
|
|
1497
|
+
stats: {
|
|
1498
|
+
matches: 1,
|
|
1499
|
+
matchedLines: 1,
|
|
1500
|
+
matchedLineNumbers: [],
|
|
1501
|
+
truncated: false,
|
|
1502
|
+
},
|
|
1503
|
+
};
|
|
1400
1504
|
}
|
|
1401
1505
|
function collectMatches(lines, regex, maxMatches) {
|
|
1402
|
-
|
|
1403
|
-
|
|
1404
|
-
|
|
1405
|
-
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
|
|
1409
|
-
|
|
1410
|
-
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
|
|
1414
|
-
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
|
|
1418
|
-
|
|
1419
|
-
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
|
|
1423
|
-
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
|
|
1427
|
-
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
|
|
1506
|
+
let matches = 0;
|
|
1507
|
+
let truncated = false;
|
|
1508
|
+
const matchedLineNumbers = [];
|
|
1509
|
+
lineLoop: for (let index = 0; index < lines.length; index += 1) {
|
|
1510
|
+
const line = lines[index] ?? "";
|
|
1511
|
+
let lineMatches = 0;
|
|
1512
|
+
regex.lastIndex = 0;
|
|
1513
|
+
let match;
|
|
1514
|
+
while ((match = regex.exec(line)) !== null) {
|
|
1515
|
+
if (match[0].length === 0) {
|
|
1516
|
+
throw new Error("Regex patterns that produce zero-width matches are not supported.");
|
|
1517
|
+
}
|
|
1518
|
+
lineMatches += 1;
|
|
1519
|
+
matches += 1;
|
|
1520
|
+
if (maxMatches !== undefined && matches >= maxMatches) {
|
|
1521
|
+
truncated = index < lines.length - 1 || regex.lastIndex < line.length;
|
|
1522
|
+
break;
|
|
1523
|
+
}
|
|
1524
|
+
}
|
|
1525
|
+
if (lineMatches > 0) {
|
|
1526
|
+
matchedLineNumbers.push(index + 1);
|
|
1527
|
+
}
|
|
1528
|
+
if (maxMatches !== undefined && matches >= maxMatches) {
|
|
1529
|
+
break lineLoop;
|
|
1530
|
+
}
|
|
1531
|
+
}
|
|
1532
|
+
return {
|
|
1533
|
+
matches,
|
|
1534
|
+
matchedLines: matchedLineNumbers.length,
|
|
1535
|
+
matchedLineNumbers,
|
|
1536
|
+
truncated,
|
|
1537
|
+
};
|
|
1434
1538
|
}
|
|
1435
1539
|
function mergeLineWindows(windows) {
|
|
1436
|
-
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
|
|
1448
|
-
|
|
1449
|
-
return merged;
|
|
1540
|
+
const sorted = windows
|
|
1541
|
+
.filter((window) => window.start <= window.end)
|
|
1542
|
+
.sort((left, right) => left.start - right.start || left.end - right.end);
|
|
1543
|
+
const merged = [];
|
|
1544
|
+
for (const window of sorted) {
|
|
1545
|
+
const previous = merged[merged.length - 1];
|
|
1546
|
+
if (!previous || window.start > previous.end + 1) {
|
|
1547
|
+
merged.push({ ...window });
|
|
1548
|
+
} else {
|
|
1549
|
+
previous.end = Math.max(previous.end, window.end);
|
|
1550
|
+
}
|
|
1551
|
+
}
|
|
1552
|
+
return merged;
|
|
1450
1553
|
}
|
|
1451
1554
|
function routeTransformedText(options) {
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1555
|
+
const caps =
|
|
1556
|
+
options.preserve === "full"
|
|
1557
|
+
? {
|
|
1558
|
+
maxLines: Number.MAX_SAFE_INTEGER,
|
|
1559
|
+
maxExcerptBytes: options.thresholds.largeOutputBytes,
|
|
1560
|
+
maxLineBytes: options.thresholds.largeOutputBytes,
|
|
1561
|
+
}
|
|
1562
|
+
: {
|
|
1563
|
+
maxLines: options.thresholds.largeOutputLines,
|
|
1564
|
+
maxExcerptBytes: options.thresholds.largeOutputBytes,
|
|
1565
|
+
maxLineBytes: options.thresholds.largeOutputBytes,
|
|
1462
1566
|
};
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
evidence
|
|
1486
|
-
|
|
1567
|
+
const bounded = assembleTextEvidence({ stream: "combined", text: options.text, caps });
|
|
1568
|
+
const outputBytes = byteLength(options.text);
|
|
1569
|
+
const outputLines = countLines(options.text);
|
|
1570
|
+
const routingStatus = bounded.compressed || bounded.fidelity === "lossy" ? "partial" : "routed";
|
|
1571
|
+
const sourceLabel = `${options.source.outputId}:${options.source.stream ?? "combined"}`;
|
|
1572
|
+
const evidence = bounded.importantLines.map((line, index) => ({
|
|
1573
|
+
id: evidenceId(options.outputId, line.lines, index),
|
|
1574
|
+
source: { kind: "vault", outputId: options.outputId, stream: "raw" },
|
|
1575
|
+
path: `${options.outputId}:raw`,
|
|
1576
|
+
lines: line.lines,
|
|
1577
|
+
excerpt: line.excerpt,
|
|
1578
|
+
why:
|
|
1579
|
+
routingStatus === "partial"
|
|
1580
|
+
? `Bounded transformed ${options.operationKind} output from source ${sourceLabel}; exact transformed content is recoverable from the vault and source lineage is preserved.`
|
|
1581
|
+
: `Transformed exact ${options.operationKind} output from source ${sourceLabel} within routing caps; source lineage is preserved.`,
|
|
1582
|
+
window: routingStatus === "partial" ? "small" : "exact",
|
|
1583
|
+
expandable: true,
|
|
1584
|
+
}));
|
|
1585
|
+
return {
|
|
1586
|
+
routingStatus,
|
|
1587
|
+
reason:
|
|
1588
|
+
routingStatus === "partial"
|
|
1589
|
+
? `Transformed output from operation ${options.operationKind} over source ${sourceLabel} (${outputBytes} bytes, ${outputLines} lines) was vaulted; bounded evidence was returned with exact recovery.`
|
|
1590
|
+
: `Transformed output from operation ${options.operationKind} over source ${sourceLabel} (${outputBytes} bytes, ${outputLines} lines) was vaulted and returned within routing caps.`,
|
|
1591
|
+
evidence,
|
|
1592
|
+
};
|
|
1487
1593
|
}
|
|
1488
1594
|
function citationEntriesForLine(line, lineNumber) {
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1515
|
-
|
|
1595
|
+
const entries = [];
|
|
1596
|
+
const inlineLinkPattern = /\[([^\]\n]+)\]\((https?:\/\/[^\s)]+)\)/gi;
|
|
1597
|
+
let inlineMatch;
|
|
1598
|
+
while ((inlineMatch = inlineLinkPattern.exec(line)) !== null) {
|
|
1599
|
+
const label = inlineMatch[1];
|
|
1600
|
+
const target = inlineMatch[2];
|
|
1601
|
+
if (label !== undefined && target !== undefined) {
|
|
1602
|
+
entries.push({ lineNumber, type: "markdown-link", label, target: trimUrl(target) });
|
|
1603
|
+
}
|
|
1604
|
+
}
|
|
1605
|
+
const citeKeyPattern = /\[@([A-Za-z0-9_.:#-]+)\]/g;
|
|
1606
|
+
let citeKeyMatch;
|
|
1607
|
+
while ((citeKeyMatch = citeKeyPattern.exec(line)) !== null) {
|
|
1608
|
+
const label = citeKeyMatch[1];
|
|
1609
|
+
if (label !== undefined) {
|
|
1610
|
+
entries.push({ lineNumber, type: "citekey", label });
|
|
1611
|
+
}
|
|
1612
|
+
}
|
|
1613
|
+
const footnote = /^\s*\[\^([^\]\n]+)\]:\s*(.+?)\s*$/.exec(line);
|
|
1614
|
+
if (footnote?.[1] !== undefined && footnote[2] !== undefined) {
|
|
1615
|
+
entries.push({ lineNumber, type: "footnote", label: footnote[1], target: footnote[2] });
|
|
1616
|
+
}
|
|
1617
|
+
const reference = /^\s*\[([^\]^][^\]\n]*)\]:\s*(.+?)\s*$/.exec(line);
|
|
1618
|
+
if (reference?.[1] !== undefined && reference[2] !== undefined) {
|
|
1619
|
+
entries.push({ lineNumber, type: "reference", label: reference[1], target: reference[2] });
|
|
1620
|
+
}
|
|
1621
|
+
return entries;
|
|
1516
1622
|
}
|
|
1517
1623
|
function trimUrl(url) {
|
|
1518
|
-
|
|
1624
|
+
return url.replace(/[.,;:!?]+$/g, "");
|
|
1519
1625
|
}
|
|
1520
1626
|
function matchGroupValue(match, group, lineNumber) {
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
if (value === undefined) {
|
|
1524
|
-
throw new Error(`Regex group ${group} did not resolve on source line ${lineNumber}.`);
|
|
1525
|
-
}
|
|
1526
|
-
return value;
|
|
1527
|
-
}
|
|
1528
|
-
const value = match.groups?.[group];
|
|
1627
|
+
if (typeof group === "number") {
|
|
1628
|
+
const value = match[group];
|
|
1529
1629
|
if (value === undefined) {
|
|
1530
|
-
|
|
1630
|
+
throw new Error(`Regex group ${group} did not resolve on source line ${lineNumber}.`);
|
|
1531
1631
|
}
|
|
1532
1632
|
return value;
|
|
1633
|
+
}
|
|
1634
|
+
const value = match.groups?.[group];
|
|
1635
|
+
if (value === undefined) {
|
|
1636
|
+
throw new Error(`Regex group ${group} did not resolve on source line ${lineNumber}.`);
|
|
1637
|
+
}
|
|
1638
|
+
return value;
|
|
1533
1639
|
}
|
|
1534
1640
|
function compareTopNEntries(left, right, sort, order) {
|
|
1535
|
-
|
|
1536
|
-
|
|
1537
|
-
|
|
1538
|
-
|
|
1539
|
-
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
return left.lineNumber - right.lineNumber;
|
|
1641
|
+
let comparison;
|
|
1642
|
+
if (sort === "numeric") {
|
|
1643
|
+
comparison = left.score - right.score;
|
|
1644
|
+
} else {
|
|
1645
|
+
const leftText = String(left.score);
|
|
1646
|
+
const rightText = String(right.score);
|
|
1647
|
+
comparison = leftText < rightText ? -1 : leftText > rightText ? 1 : 0;
|
|
1648
|
+
}
|
|
1649
|
+
if (comparison !== 0) {
|
|
1650
|
+
return order === "desc" ? -comparison : comparison;
|
|
1651
|
+
}
|
|
1652
|
+
return left.lineNumber - right.lineNumber;
|
|
1548
1653
|
}
|
|
1549
1654
|
function parseJsonPointer(pointer) {
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
|
|
1557
|
-
|
|
1558
|
-
|
|
1559
|
-
|
|
1560
|
-
|
|
1561
|
-
|
|
1562
|
-
|
|
1563
|
-
|
|
1655
|
+
if (pointer === "") {
|
|
1656
|
+
return { ok: true, segments: [] };
|
|
1657
|
+
}
|
|
1658
|
+
if (!pointer.startsWith("/")) {
|
|
1659
|
+
return { ok: false, message: "JSON pointer must be empty or start with /." };
|
|
1660
|
+
}
|
|
1661
|
+
const segments = [];
|
|
1662
|
+
for (const rawSegment of pointer.slice(1).split("/")) {
|
|
1663
|
+
if (/~(?![01])/.test(rawSegment)) {
|
|
1664
|
+
return { ok: false, message: "JSON pointer contains an invalid escape; use ~0 for ~ and ~1 for /." };
|
|
1665
|
+
}
|
|
1666
|
+
segments.push({ kind: "property", key: rawSegment.replace(/~1/g, "/").replace(/~0/g, "~") });
|
|
1667
|
+
}
|
|
1668
|
+
return { ok: true, segments };
|
|
1564
1669
|
}
|
|
1565
1670
|
function parseJsonPath(path) {
|
|
1566
|
-
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
|
|
1571
|
-
|
|
1572
|
-
|
|
1573
|
-
|
|
1574
|
-
|
|
1575
|
-
|
|
1576
|
-
|
|
1577
|
-
|
|
1578
|
-
|
|
1579
|
-
|
|
1580
|
-
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
|
|
1591
|
-
|
|
1592
|
-
|
|
1593
|
-
|
|
1671
|
+
if (path.length === 0 || path[0] !== "$") {
|
|
1672
|
+
return { ok: false, message: "JSON path must start with $." };
|
|
1673
|
+
}
|
|
1674
|
+
const segments = [];
|
|
1675
|
+
let index = 1;
|
|
1676
|
+
while (index < path.length) {
|
|
1677
|
+
const char = path[index];
|
|
1678
|
+
if (char === ".") {
|
|
1679
|
+
const parsed = parseJsonPathProperty(path, index + 1);
|
|
1680
|
+
if (!parsed.ok) {
|
|
1681
|
+
return parsed;
|
|
1682
|
+
}
|
|
1683
|
+
segments.push({ kind: "property", key: parsed.key });
|
|
1684
|
+
index = parsed.nextIndex;
|
|
1685
|
+
continue;
|
|
1686
|
+
}
|
|
1687
|
+
if (char === "[") {
|
|
1688
|
+
const parsed = parseJsonPathBracket(path, index + 1);
|
|
1689
|
+
if (!parsed.ok) {
|
|
1690
|
+
return parsed;
|
|
1691
|
+
}
|
|
1692
|
+
segments.push(parsed.segment);
|
|
1693
|
+
index = parsed.nextIndex;
|
|
1694
|
+
continue;
|
|
1695
|
+
}
|
|
1696
|
+
return { ok: false, message: `Unexpected token ${char} at offset ${index}.` };
|
|
1697
|
+
}
|
|
1698
|
+
return { ok: true, segments };
|
|
1594
1699
|
}
|
|
1595
1700
|
function parseJsonPathProperty(path, startIndex) {
|
|
1596
|
-
|
|
1597
|
-
|
|
1598
|
-
|
|
1599
|
-
|
|
1600
|
-
|
|
1701
|
+
const match = /^[A-Za-z_$][A-Za-z0-9_$-]*/.exec(path.slice(startIndex));
|
|
1702
|
+
if (!match?.[0]) {
|
|
1703
|
+
return { ok: false, message: `Expected property name after . at offset ${startIndex - 1}.` };
|
|
1704
|
+
}
|
|
1705
|
+
return { ok: true, key: match[0], nextIndex: startIndex + match[0].length };
|
|
1601
1706
|
}
|
|
1602
1707
|
function parseJsonPathBracket(path, startIndex) {
|
|
1603
|
-
|
|
1604
|
-
|
|
1605
|
-
|
|
1606
|
-
|
|
1607
|
-
|
|
1608
|
-
|
|
1609
|
-
|
|
1610
|
-
|
|
1611
|
-
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1708
|
+
const first = path[startIndex];
|
|
1709
|
+
if (first === '"') {
|
|
1710
|
+
return parseJsonPathQuotedProperty(path, startIndex);
|
|
1711
|
+
}
|
|
1712
|
+
if (first === "'") {
|
|
1713
|
+
return { ok: false, message: `Use double-quoted bracket properties at offset ${startIndex}.` };
|
|
1714
|
+
}
|
|
1715
|
+
const closeIndex = path.indexOf("]", startIndex);
|
|
1716
|
+
if (closeIndex === -1) {
|
|
1717
|
+
return { ok: false, message: `Expected ] for bracket selector at offset ${startIndex - 1}.` };
|
|
1718
|
+
}
|
|
1719
|
+
const indexText = path.slice(startIndex, closeIndex);
|
|
1720
|
+
if (!/^(0|[1-9][0-9]*)$/.test(indexText)) {
|
|
1721
|
+
return { ok: false, message: `Expected non-negative array index in bracket selector at offset ${startIndex - 1}.` };
|
|
1722
|
+
}
|
|
1723
|
+
return { ok: true, segment: { kind: "index", index: Number(indexText) }, nextIndex: closeIndex + 1 };
|
|
1619
1724
|
}
|
|
1620
1725
|
function parseJsonPathQuotedProperty(path, quoteIndex) {
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
|
|
1634
|
-
|
|
1635
|
-
|
|
1636
|
-
|
|
1637
|
-
|
|
1638
|
-
|
|
1639
|
-
|
|
1640
|
-
|
|
1641
|
-
|
|
1642
|
-
|
|
1643
|
-
|
|
1644
|
-
|
|
1645
|
-
|
|
1646
|
-
|
|
1647
|
-
|
|
1648
|
-
|
|
1649
|
-
return { ok: false, message: `Unterminated quoted property at offset ${quoteIndex}.` };
|
|
1726
|
+
let escaped = false;
|
|
1727
|
+
for (let index = quoteIndex + 1; index < path.length; index += 1) {
|
|
1728
|
+
const char = path[index];
|
|
1729
|
+
if (escaped) {
|
|
1730
|
+
escaped = false;
|
|
1731
|
+
continue;
|
|
1732
|
+
}
|
|
1733
|
+
if (char === "\\") {
|
|
1734
|
+
escaped = true;
|
|
1735
|
+
continue;
|
|
1736
|
+
}
|
|
1737
|
+
if (char === '"') {
|
|
1738
|
+
if (path[index + 1] !== "]") {
|
|
1739
|
+
return { ok: false, message: `Expected ] after quoted property at offset ${index}.` };
|
|
1740
|
+
}
|
|
1741
|
+
const literal = path.slice(quoteIndex, index + 1);
|
|
1742
|
+
try {
|
|
1743
|
+
const key = JSON.parse(literal);
|
|
1744
|
+
if (typeof key !== "string") {
|
|
1745
|
+
return { ok: false, message: "Expected quoted JSON path property to decode to a string." };
|
|
1746
|
+
}
|
|
1747
|
+
return { ok: true, segment: { kind: "property", key }, nextIndex: index + 2 };
|
|
1748
|
+
} catch (error) {
|
|
1749
|
+
return { ok: false, message: `Invalid quoted property string: ${errorMessage(error)}` };
|
|
1750
|
+
}
|
|
1751
|
+
}
|
|
1752
|
+
}
|
|
1753
|
+
return { ok: false, message: `Unterminated quoted property at offset ${quoteIndex}.` };
|
|
1650
1754
|
}
|
|
1651
1755
|
function resolveJsonSelector(value, segments) {
|
|
1652
|
-
|
|
1653
|
-
|
|
1654
|
-
|
|
1655
|
-
|
|
1656
|
-
|
|
1657
|
-
|
|
1658
|
-
|
|
1659
|
-
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
|
|
1756
|
+
let current = value;
|
|
1757
|
+
for (const segment of segments) {
|
|
1758
|
+
if (segment.kind === "index") {
|
|
1759
|
+
if (!Array.isArray(current)) {
|
|
1760
|
+
return { ok: false, message: `array index ${segment.index} cannot be applied to ${jsonValueType(current)}.` };
|
|
1761
|
+
}
|
|
1762
|
+
if (segment.index >= current.length) {
|
|
1763
|
+
return { ok: false, message: `array index ${segment.index} is outside length ${current.length}.` };
|
|
1764
|
+
}
|
|
1765
|
+
current = current[segment.index];
|
|
1766
|
+
continue;
|
|
1767
|
+
}
|
|
1768
|
+
if (Array.isArray(current)) {
|
|
1769
|
+
if (!isArrayIndexSegment(segment.key)) {
|
|
1770
|
+
return { ok: false, message: `array source requires numeric pointer segment, got ${segment.key}.` };
|
|
1771
|
+
}
|
|
1772
|
+
const arrayIndex = Number(segment.key);
|
|
1773
|
+
if (arrayIndex >= current.length) {
|
|
1774
|
+
return { ok: false, message: `array index ${arrayIndex} is outside length ${current.length}.` };
|
|
1775
|
+
}
|
|
1776
|
+
current = current[arrayIndex];
|
|
1777
|
+
continue;
|
|
1778
|
+
}
|
|
1779
|
+
if (!isJsonObject(current)) {
|
|
1780
|
+
return { ok: false, message: `property ${segment.key} cannot be applied to ${jsonValueType(current)}.` };
|
|
1781
|
+
}
|
|
1782
|
+
if (!Object.prototype.hasOwnProperty.call(current, segment.key)) {
|
|
1783
|
+
return { ok: false, message: `property ${segment.key} is not present.` };
|
|
1784
|
+
}
|
|
1785
|
+
current = current[segment.key];
|
|
1786
|
+
}
|
|
1787
|
+
return { ok: true, value: current };
|
|
1684
1788
|
}
|
|
1685
1789
|
function isJsonObject(value) {
|
|
1686
|
-
|
|
1790
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
1687
1791
|
}
|
|
1688
1792
|
function isArrayIndexSegment(value) {
|
|
1689
|
-
|
|
1793
|
+
return /^(0|[1-9][0-9]*)$/.test(value);
|
|
1690
1794
|
}
|
|
1691
1795
|
function jsonValueType(value) {
|
|
1692
|
-
|
|
1693
|
-
|
|
1694
|
-
|
|
1695
|
-
|
|
1696
|
-
|
|
1697
|
-
|
|
1698
|
-
|
|
1796
|
+
if (value === null) {
|
|
1797
|
+
return "null";
|
|
1798
|
+
}
|
|
1799
|
+
if (Array.isArray(value)) {
|
|
1800
|
+
return "array";
|
|
1801
|
+
}
|
|
1802
|
+
return typeof value;
|
|
1699
1803
|
}
|
|
1700
1804
|
function effectiveScriptTransformConfig(config) {
|
|
1701
|
-
|
|
1702
|
-
return {
|
|
1703
|
-
...DEFAULT_SCRIPT_TRANSFORM_CONFIG,
|
|
1704
|
-
languages: [...DEFAULT_SCRIPT_TRANSFORM_CONFIG.languages],
|
|
1705
|
-
limits: { ...DEFAULT_SCRIPT_TRANSFORM_CONFIG.limits },
|
|
1706
|
-
};
|
|
1707
|
-
}
|
|
1805
|
+
if (config === undefined) {
|
|
1708
1806
|
return {
|
|
1709
|
-
|
|
1710
|
-
|
|
1711
|
-
|
|
1807
|
+
...DEFAULT_SCRIPT_TRANSFORM_CONFIG,
|
|
1808
|
+
languages: [...DEFAULT_SCRIPT_TRANSFORM_CONFIG.languages],
|
|
1809
|
+
limits: { ...DEFAULT_SCRIPT_TRANSFORM_CONFIG.limits },
|
|
1712
1810
|
};
|
|
1811
|
+
}
|
|
1812
|
+
return {
|
|
1813
|
+
...config,
|
|
1814
|
+
languages: [...config.languages],
|
|
1815
|
+
limits: { ...config.limits },
|
|
1816
|
+
};
|
|
1713
1817
|
}
|
|
1714
1818
|
function effectiveScriptLimits(config, input) {
|
|
1715
|
-
|
|
1716
|
-
|
|
1717
|
-
|
|
1718
|
-
|
|
1719
|
-
|
|
1819
|
+
return {
|
|
1820
|
+
timeoutMs: boundedScriptLimit(input?.timeoutMs, config.limits.timeoutMs, MAX_SCRIPT_TRANSFORM_LIMITS.timeoutMs),
|
|
1821
|
+
maxInputBytes: boundedScriptLimit(
|
|
1822
|
+
input?.maxInputBytes,
|
|
1823
|
+
config.limits.maxInputBytes,
|
|
1824
|
+
MAX_SCRIPT_TRANSFORM_LIMITS.maxInputBytes,
|
|
1825
|
+
),
|
|
1826
|
+
maxOutputBytes: boundedScriptLimit(
|
|
1827
|
+
input?.maxOutputBytes,
|
|
1828
|
+
config.limits.maxOutputBytes,
|
|
1829
|
+
MAX_SCRIPT_TRANSFORM_LIMITS.maxOutputBytes,
|
|
1830
|
+
),
|
|
1831
|
+
};
|
|
1720
1832
|
}
|
|
1721
1833
|
function boundedScriptLimit(value, configured, max) {
|
|
1722
|
-
|
|
1723
|
-
|
|
1724
|
-
|
|
1725
|
-
|
|
1726
|
-
|
|
1834
|
+
const configuredLimit = Math.min(configured, max);
|
|
1835
|
+
if (!Number.isInteger(value) || Number(value) <= 0) {
|
|
1836
|
+
return configuredLimit;
|
|
1837
|
+
}
|
|
1838
|
+
return Math.min(Number(value), configuredLimit);
|
|
1727
1839
|
}
|
|
1728
1840
|
function resolveSourceStream(record, requested) {
|
|
1729
|
-
|
|
1730
|
-
|
|
1731
|
-
|
|
1732
|
-
|
|
1733
|
-
|
|
1734
|
-
|
|
1735
|
-
|
|
1736
|
-
|
|
1737
|
-
|
|
1738
|
-
|
|
1739
|
-
|
|
1740
|
-
|
|
1741
|
-
|
|
1742
|
-
|
|
1743
|
-
|
|
1841
|
+
if (record.kind === "command") {
|
|
1842
|
+
const stream = requested ?? "combined";
|
|
1843
|
+
if (stream === "raw") {
|
|
1844
|
+
return { ok: false, message: "Command vault sources support stdout, stderr, or combined streams, not raw." };
|
|
1845
|
+
}
|
|
1846
|
+
return { ok: true, stream };
|
|
1847
|
+
}
|
|
1848
|
+
if (record.kind === "text") {
|
|
1849
|
+
const stream = requested ?? "raw";
|
|
1850
|
+
if (stream !== "raw") {
|
|
1851
|
+
return { ok: false, message: "Text vault sources support only the raw stream." };
|
|
1852
|
+
}
|
|
1853
|
+
return { ok: true, stream };
|
|
1854
|
+
}
|
|
1855
|
+
return {
|
|
1856
|
+
ok: false,
|
|
1857
|
+
message: "Repo file reference vault records store metadata only and cannot be used as transform text sources.",
|
|
1858
|
+
};
|
|
1744
1859
|
}
|
|
1745
1860
|
function lineageForSource(record, operation) {
|
|
1746
|
-
|
|
1747
|
-
|
|
1748
|
-
|
|
1749
|
-
|
|
1750
|
-
|
|
1751
|
-
|
|
1861
|
+
return {
|
|
1862
|
+
sourceRecordIds: [record.recordId],
|
|
1863
|
+
sourceOutputIds: [record.outputId],
|
|
1864
|
+
operation: operation.kind === "script" ? `script:${operation.language}` : operation.kind,
|
|
1865
|
+
operationHash: operationHash(operation),
|
|
1866
|
+
};
|
|
1752
1867
|
}
|
|
1753
1868
|
function lineageForScriptInput(input, limits) {
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
|
|
1758
|
-
|
|
1869
|
+
return {
|
|
1870
|
+
sourceOutputIds: input.sources.map((source) => source.outputId),
|
|
1871
|
+
operation: `script:${input.operation.language}`,
|
|
1872
|
+
operationHash: operationHashForScript(input.operation, input.sources, limits ?? input.limits),
|
|
1873
|
+
};
|
|
1759
1874
|
}
|
|
1760
1875
|
function lineageForResolvedScriptSources(sources, input, limits) {
|
|
1761
|
-
|
|
1762
|
-
|
|
1763
|
-
|
|
1764
|
-
|
|
1765
|
-
|
|
1766
|
-
|
|
1876
|
+
return {
|
|
1877
|
+
sourceRecordIds: sources.map((source) => source.recordId),
|
|
1878
|
+
sourceOutputIds: sources.map((source) => source.outputId),
|
|
1879
|
+
operation: `script:${input.operation.language}`,
|
|
1880
|
+
operationHash: operationHashForScript(input.operation, input.sources, limits, sources),
|
|
1881
|
+
};
|
|
1767
1882
|
}
|
|
1768
1883
|
function lineageFromInput(input) {
|
|
1769
|
-
|
|
1770
|
-
|
|
1771
|
-
|
|
1772
|
-
|
|
1773
|
-
|
|
1774
|
-
|
|
1775
|
-
|
|
1776
|
-
|
|
1777
|
-
|
|
1778
|
-
|
|
1779
|
-
|
|
1780
|
-
|
|
1781
|
-
|
|
1782
|
-
|
|
1783
|
-
|
|
1784
|
-
|
|
1785
|
-
|
|
1786
|
-
|
|
1787
|
-
|
|
1788
|
-
|
|
1789
|
-
|
|
1790
|
-
|
|
1884
|
+
if (!isRecord(input) || !isRecord(input.operation)) {
|
|
1885
|
+
return undefined;
|
|
1886
|
+
}
|
|
1887
|
+
if (input.operation.kind === "script" && Array.isArray(input.sources)) {
|
|
1888
|
+
const sources = input.sources.filter(isRecord);
|
|
1889
|
+
const outputIds = sources.map((source) => source.outputId).filter((outputId) => typeof outputId === "string");
|
|
1890
|
+
return {
|
|
1891
|
+
...(outputIds.length > 0 ? { sourceOutputIds: outputIds } : {}),
|
|
1892
|
+
operation: typeof input.operation.language === "string" ? `script:${input.operation.language}` : "script",
|
|
1893
|
+
};
|
|
1894
|
+
}
|
|
1895
|
+
if (!isRecord(input.source)) {
|
|
1896
|
+
return undefined;
|
|
1897
|
+
}
|
|
1898
|
+
const lineage = {};
|
|
1899
|
+
if (typeof input.source.outputId === "string") {
|
|
1900
|
+
lineage.sourceOutputIds = [input.source.outputId];
|
|
1901
|
+
}
|
|
1902
|
+
if (typeof input.operation.kind === "string") {
|
|
1903
|
+
lineage.operation = input.operation.kind;
|
|
1904
|
+
}
|
|
1905
|
+
return Object.keys(lineage).length > 0 ? lineage : undefined;
|
|
1791
1906
|
}
|
|
1792
1907
|
function operationSummary(operation) {
|
|
1793
|
-
|
|
1794
|
-
|
|
1795
|
-
|
|
1796
|
-
|
|
1797
|
-
return {
|
|
1798
|
-
kind: operation.kind,
|
|
1799
|
-
pattern: operation.pattern,
|
|
1800
|
-
...(operation.flags !== undefined ? { flags: normalizeRegexFlags(operation.flags) } : {}),
|
|
1801
|
-
contextLines: operation.contextLines ?? DEFAULT_CONTEXT_LINES,
|
|
1802
|
-
maxMatches: operation.maxMatches ?? DEFAULT_MAX_MATCHES,
|
|
1803
|
-
};
|
|
1804
|
-
}
|
|
1805
|
-
if (operation.kind === "countMatches") {
|
|
1806
|
-
return {
|
|
1807
|
-
kind: operation.kind,
|
|
1808
|
-
pattern: operation.pattern,
|
|
1809
|
-
...(operation.flags !== undefined ? { flags: normalizeRegexFlags(operation.flags) } : {}),
|
|
1810
|
-
};
|
|
1811
|
-
}
|
|
1812
|
-
if (operation.kind === "groupByRegex") {
|
|
1813
|
-
return {
|
|
1814
|
-
kind: operation.kind,
|
|
1815
|
-
pattern: operation.pattern,
|
|
1816
|
-
...(operation.flags !== undefined ? { flags: normalizeRegexFlags(operation.flags) } : {}),
|
|
1817
|
-
group: operation.group ?? 1,
|
|
1818
|
-
maxGroups: operation.maxGroups ?? DEFAULT_MAX_GROUPS,
|
|
1819
|
-
maxLinesPerGroup: operation.maxLinesPerGroup ?? DEFAULT_MAX_LINES_PER_GROUP,
|
|
1820
|
-
};
|
|
1821
|
-
}
|
|
1822
|
-
if (operation.kind === "dedupe") {
|
|
1823
|
-
return {
|
|
1824
|
-
kind: operation.kind,
|
|
1825
|
-
trim: operation.trim ?? false,
|
|
1826
|
-
caseSensitive: operation.caseSensitive ?? true,
|
|
1827
|
-
maxLines: operation.maxLines ?? DEFAULT_MAX_DEDUPE_LINES,
|
|
1828
|
-
};
|
|
1829
|
-
}
|
|
1830
|
-
if (operation.kind === "topN") {
|
|
1831
|
-
return {
|
|
1832
|
-
kind: operation.kind,
|
|
1833
|
-
limit: operation.limit,
|
|
1834
|
-
...(operation.pattern !== undefined ? { pattern: operation.pattern } : {}),
|
|
1835
|
-
...(operation.flags !== undefined ? { flags: normalizeRegexFlags(operation.flags) } : {}),
|
|
1836
|
-
...(operation.group !== undefined ? { group: operation.group } : {}),
|
|
1837
|
-
sort: operation.sort ?? "text",
|
|
1838
|
-
order: operation.order ?? "asc",
|
|
1839
|
-
};
|
|
1840
|
-
}
|
|
1841
|
-
if (operation.kind === "extractUrls") {
|
|
1842
|
-
return {
|
|
1843
|
-
kind: operation.kind,
|
|
1844
|
-
dedupe: operation.dedupe ?? false,
|
|
1845
|
-
maxMatches: operation.maxMatches ?? DEFAULT_MAX_EXTRACT_MATCHES,
|
|
1846
|
-
};
|
|
1847
|
-
}
|
|
1848
|
-
if (operation.kind === "extractCitations") {
|
|
1849
|
-
return {
|
|
1850
|
-
kind: operation.kind,
|
|
1851
|
-
maxMatches: operation.maxMatches ?? DEFAULT_MAX_EXTRACT_MATCHES,
|
|
1852
|
-
};
|
|
1853
|
-
}
|
|
1854
|
-
if (operation.kind === "lineStats" || operation.kind === "sizeStats") {
|
|
1855
|
-
return { kind: operation.kind };
|
|
1856
|
-
}
|
|
1908
|
+
if (operation.kind === "script") {
|
|
1909
|
+
return scriptOperationSummary(operation);
|
|
1910
|
+
}
|
|
1911
|
+
if (operation.kind === "regexFilter") {
|
|
1857
1912
|
return {
|
|
1858
|
-
|
|
1859
|
-
|
|
1860
|
-
|
|
1913
|
+
kind: operation.kind,
|
|
1914
|
+
pattern: operation.pattern,
|
|
1915
|
+
...(operation.flags !== undefined ? { flags: normalizeRegexFlags(operation.flags) } : {}),
|
|
1916
|
+
contextLines: operation.contextLines ?? DEFAULT_CONTEXT_LINES,
|
|
1917
|
+
maxMatches: operation.maxMatches ?? DEFAULT_MAX_MATCHES,
|
|
1861
1918
|
};
|
|
1919
|
+
}
|
|
1920
|
+
if (operation.kind === "countMatches") {
|
|
1921
|
+
return {
|
|
1922
|
+
kind: operation.kind,
|
|
1923
|
+
pattern: operation.pattern,
|
|
1924
|
+
...(operation.flags !== undefined ? { flags: normalizeRegexFlags(operation.flags) } : {}),
|
|
1925
|
+
};
|
|
1926
|
+
}
|
|
1927
|
+
if (operation.kind === "groupByRegex") {
|
|
1928
|
+
return {
|
|
1929
|
+
kind: operation.kind,
|
|
1930
|
+
pattern: operation.pattern,
|
|
1931
|
+
...(operation.flags !== undefined ? { flags: normalizeRegexFlags(operation.flags) } : {}),
|
|
1932
|
+
group: operation.group ?? 1,
|
|
1933
|
+
maxGroups: operation.maxGroups ?? DEFAULT_MAX_GROUPS,
|
|
1934
|
+
maxLinesPerGroup: operation.maxLinesPerGroup ?? DEFAULT_MAX_LINES_PER_GROUP,
|
|
1935
|
+
};
|
|
1936
|
+
}
|
|
1937
|
+
if (operation.kind === "dedupe") {
|
|
1938
|
+
return {
|
|
1939
|
+
kind: operation.kind,
|
|
1940
|
+
trim: operation.trim ?? false,
|
|
1941
|
+
caseSensitive: operation.caseSensitive ?? true,
|
|
1942
|
+
maxLines: operation.maxLines ?? DEFAULT_MAX_DEDUPE_LINES,
|
|
1943
|
+
};
|
|
1944
|
+
}
|
|
1945
|
+
if (operation.kind === "topN") {
|
|
1946
|
+
return {
|
|
1947
|
+
kind: operation.kind,
|
|
1948
|
+
limit: operation.limit,
|
|
1949
|
+
...(operation.pattern !== undefined ? { pattern: operation.pattern } : {}),
|
|
1950
|
+
...(operation.flags !== undefined ? { flags: normalizeRegexFlags(operation.flags) } : {}),
|
|
1951
|
+
...(operation.group !== undefined ? { group: operation.group } : {}),
|
|
1952
|
+
sort: operation.sort ?? "text",
|
|
1953
|
+
order: operation.order ?? "asc",
|
|
1954
|
+
};
|
|
1955
|
+
}
|
|
1956
|
+
if (operation.kind === "extractUrls") {
|
|
1957
|
+
return {
|
|
1958
|
+
kind: operation.kind,
|
|
1959
|
+
dedupe: operation.dedupe ?? false,
|
|
1960
|
+
maxMatches: operation.maxMatches ?? DEFAULT_MAX_EXTRACT_MATCHES,
|
|
1961
|
+
};
|
|
1962
|
+
}
|
|
1963
|
+
if (operation.kind === "extractCitations") {
|
|
1964
|
+
return {
|
|
1965
|
+
kind: operation.kind,
|
|
1966
|
+
maxMatches: operation.maxMatches ?? DEFAULT_MAX_EXTRACT_MATCHES,
|
|
1967
|
+
};
|
|
1968
|
+
}
|
|
1969
|
+
if (operation.kind === "lineStats" || operation.kind === "sizeStats") {
|
|
1970
|
+
return { kind: operation.kind };
|
|
1971
|
+
}
|
|
1972
|
+
return {
|
|
1973
|
+
kind: operation.kind,
|
|
1974
|
+
...(operation.pointer !== undefined ? { pointer: operation.pointer } : {}),
|
|
1975
|
+
...(operation.path !== undefined ? { path: operation.path } : {}),
|
|
1976
|
+
};
|
|
1862
1977
|
}
|
|
1863
1978
|
function operationHash(operation) {
|
|
1864
|
-
|
|
1979
|
+
return `sha256_${hash(JSON.stringify(operationSummary(operation)))}`;
|
|
1865
1980
|
}
|
|
1866
1981
|
function operationHashForScript(operation, sources, limits, resolvedSources = []) {
|
|
1867
|
-
|
|
1868
|
-
|
|
1869
|
-
|
|
1870
|
-
|
|
1871
|
-
|
|
1872
|
-
|
|
1873
|
-
|
|
1874
|
-
|
|
1875
|
-
|
|
1876
|
-
|
|
1877
|
-
|
|
1878
|
-
|
|
1879
|
-
|
|
1880
|
-
|
|
1881
|
-
|
|
1982
|
+
return `sha256_${hash(
|
|
1983
|
+
JSON.stringify({
|
|
1984
|
+
schemaVersion: 1,
|
|
1985
|
+
operation: scriptOperationSummary(operation),
|
|
1986
|
+
sources: sources.map((source) => ({ alias: source.alias, outputId: source.outputId, stream: source.stream })),
|
|
1987
|
+
resolvedSources: resolvedSources.map((source) => ({
|
|
1988
|
+
alias: source.alias,
|
|
1989
|
+
outputId: source.outputId,
|
|
1990
|
+
recordId: source.recordId,
|
|
1991
|
+
stream: source.stream,
|
|
1992
|
+
bytes: source.bytes,
|
|
1993
|
+
contentHashSha256: source.contentHashSha256,
|
|
1994
|
+
textSha256: source.textSha256,
|
|
1995
|
+
})),
|
|
1996
|
+
limits,
|
|
1997
|
+
}),
|
|
1998
|
+
)}`;
|
|
1882
1999
|
}
|
|
1883
2000
|
function scriptOperationSummary(operation) {
|
|
1884
|
-
|
|
1885
|
-
|
|
1886
|
-
|
|
1887
|
-
|
|
1888
|
-
|
|
1889
|
-
|
|
2001
|
+
return {
|
|
2002
|
+
kind: "script",
|
|
2003
|
+
language: operation.language,
|
|
2004
|
+
codeSha256: `sha256_${hash(operation.code)}`,
|
|
2005
|
+
...(operation.label !== undefined ? { label: operation.label } : {}),
|
|
2006
|
+
};
|
|
1890
2007
|
}
|
|
1891
2008
|
function normalizeRegexFlags(flags) {
|
|
1892
|
-
|
|
2009
|
+
return [...new Set((flags ?? "").replace(/g/g, "").split(""))].sort().join("");
|
|
1893
2010
|
}
|
|
1894
2011
|
function ensureGlobalFlag(flags) {
|
|
1895
|
-
|
|
2012
|
+
return flags.includes("g") ? flags : `${flags}g`;
|
|
1896
2013
|
}
|
|
1897
2014
|
function formatPattern(compiled) {
|
|
1898
|
-
|
|
2015
|
+
return `/${compiled.displayPattern}/${compiled.flags}`;
|
|
1899
2016
|
}
|
|
1900
2017
|
function sourceStreamLabel(sourceLabel) {
|
|
1901
|
-
|
|
1902
|
-
|
|
2018
|
+
const [, stream] = sourceLabel.split(":");
|
|
2019
|
+
return stream ?? "combined";
|
|
1903
2020
|
}
|
|
1904
2021
|
function transformValidationFailureWithOptionalLineage(options) {
|
|
1905
|
-
|
|
1906
|
-
|
|
1907
|
-
|
|
1908
|
-
|
|
1909
|
-
|
|
1910
|
-
|
|
1911
|
-
|
|
1912
|
-
|
|
1913
|
-
|
|
2022
|
+
const failureOptions = {
|
|
2023
|
+
message: options.message,
|
|
2024
|
+
preserve: options.preserve,
|
|
2025
|
+
decisionSeed: options.decisionSeed,
|
|
2026
|
+
};
|
|
2027
|
+
if (options.lineage !== undefined) {
|
|
2028
|
+
return transformValidationFailure({ ...failureOptions, lineage: options.lineage });
|
|
2029
|
+
}
|
|
2030
|
+
return transformValidationFailure(failureOptions);
|
|
1914
2031
|
}
|
|
1915
2032
|
function transformSourceUnavailableFailureWithOptionalLineage(options) {
|
|
1916
|
-
|
|
1917
|
-
|
|
1918
|
-
|
|
1919
|
-
|
|
1920
|
-
|
|
1921
|
-
|
|
1922
|
-
|
|
1923
|
-
|
|
1924
|
-
|
|
2033
|
+
const failureOptions = {
|
|
2034
|
+
message: options.message,
|
|
2035
|
+
preserve: options.preserve,
|
|
2036
|
+
decisionSeed: options.decisionSeed,
|
|
2037
|
+
};
|
|
2038
|
+
if (options.lineage !== undefined) {
|
|
2039
|
+
return transformSourceUnavailableFailure({ ...failureOptions, lineage: options.lineage });
|
|
2040
|
+
}
|
|
2041
|
+
return transformSourceUnavailableFailure(failureOptions);
|
|
1925
2042
|
}
|
|
1926
2043
|
function validationMessage(issues) {
|
|
1927
|
-
|
|
2044
|
+
return `Invalid transform input: ${issues.map((issue) => `${issue.path} ${issue.message}`).join("; ")}`;
|
|
1928
2045
|
}
|
|
1929
2046
|
function isScriptTransformInput(input) {
|
|
1930
|
-
|
|
2047
|
+
return input.operation.kind === "script";
|
|
1931
2048
|
}
|
|
1932
2049
|
function isOutputStream(value) {
|
|
1933
|
-
|
|
2050
|
+
return value === "stdout" || value === "stderr" || value === "combined" || value === "raw";
|
|
1934
2051
|
}
|
|
1935
2052
|
function isStringIn(value, allowed) {
|
|
1936
|
-
|
|
2053
|
+
return typeof value === "string" && allowed.includes(value);
|
|
1937
2054
|
}
|
|
1938
2055
|
function isRecord(value) {
|
|
1939
|
-
|
|
2056
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
1940
2057
|
}
|
|
1941
2058
|
function evidenceId(outputId, lines, index) {
|
|
1942
|
-
|
|
2059
|
+
return `ev_${hash(`${outputId}:${lines}:${index}`).slice(0, 16)}`;
|
|
1943
2060
|
}
|
|
1944
2061
|
function decisionId(...parts) {
|
|
1945
|
-
|
|
2062
|
+
return `ffdec_${hash(parts.join("\0")).slice(0, 16)}`;
|
|
1946
2063
|
}
|
|
1947
2064
|
function hash(value) {
|
|
1948
|
-
|
|
2065
|
+
return createHash("sha256").update(value).digest("hex");
|
|
1949
2066
|
}
|
|
1950
2067
|
function hashText(value) {
|
|
1951
|
-
|
|
2068
|
+
return createHash("sha256").update(value).digest("hex");
|
|
1952
2069
|
}
|
|
1953
2070
|
function errorMessage(error) {
|
|
1954
|
-
|
|
2071
|
+
return error instanceof Error ? error.message : String(error);
|
|
1955
2072
|
}
|