@hassangameryt/freeflow 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/.codex-plugin/plugin.json +1 -1
- package/README.md +148 -154
- package/capabilities/cognitive-routing/SKILL.md +229 -0
- package/capabilities/context-virtualization/SKILL.md +103 -0
- package/capabilities/conversation-history/SKILL.md +53 -0
- package/command-surface.json +6 -13
- package/hooks/freeflow-runtime-context.mjs +35 -91
- package/hooks/tests/check-runtime-context-hook.sh +56 -39
- package/package.json +24 -16
- package/pi-extension/dist/cognitive-routing/commands.js +96 -0
- package/pi-extension/dist/cognitive-routing/config.js +236 -0
- package/pi-extension/dist/cognitive-routing/controller.js +862 -0
- package/pi-extension/dist/cognitive-routing/history.js +439 -0
- package/pi-extension/dist/cognitive-routing/host.js +126 -0
- package/pi-extension/dist/cognitive-routing/runtime.js +43 -0
- package/pi-extension/dist/cognitive-routing/tool.js +211 -0
- package/pi-extension/dist/cognitive-routing/types.js +6 -0
- package/pi-extension/dist/context-virtualization/commands.js +89 -0
- package/pi-extension/dist/context-virtualization/projector.js +42 -0
- package/pi-extension/dist/context-virtualization/runtime.js +356 -0
- package/pi-extension/dist/context-virtualization/state.js +57 -0
- package/pi-extension/dist/context-virtualization/types.js +10 -0
- package/pi-extension/dist/conversation-history/passages.js +66 -0
- package/pi-extension/dist/conversation-history/ranking.js +105 -0
- package/pi-extension/dist/conversation-history/runtime.js +226 -0
- package/pi-extension/dist/conversation-history/search.js +56 -0
- package/pi-extension/dist/conversation-history/types.js +1 -0
- package/pi-extension/dist/freeflow-context/resolver.js +120 -0
- package/pi-extension/dist/freeflow-context/runtime.js +80 -0
- package/pi-extension/dist/freeflow-context/source-projector.js +95 -0
- package/pi-extension/dist/freeflow-context/tool.js +308 -0
- package/pi-extension/dist/freeflow-context/types.js +12 -0
- package/pi-extension/dist/freeflow-context/visibility.js +28 -0
- package/pi-extension/dist/index.js +516 -98
- package/pi-extension/dist/runtime/runtime-context.js +318 -271
- package/pi-extension/dist/runtime/runtime-identity.js +5 -0
- package/pi-extension/dist/settings/settings-tui.js +78 -0
- package/pi-extension/dist/settings/settings-ui.js +593 -563
- package/pi-extension/dist/ui/text-component.js +57 -0
- package/pi-extension/src/cognitive-routing/commands.ts +119 -0
- package/pi-extension/src/cognitive-routing/config.ts +307 -0
- package/pi-extension/src/cognitive-routing/controller.ts +1118 -0
- package/pi-extension/src/cognitive-routing/history.ts +638 -0
- package/pi-extension/src/cognitive-routing/host.ts +172 -0
- package/pi-extension/src/cognitive-routing/runtime.ts +57 -0
- package/pi-extension/src/cognitive-routing/tool.ts +272 -0
- package/pi-extension/src/cognitive-routing/types.ts +89 -0
- package/pi-extension/src/context-virtualization/commands.ts +111 -0
- package/pi-extension/src/context-virtualization/projector.ts +62 -0
- package/pi-extension/src/context-virtualization/runtime.ts +406 -0
- package/pi-extension/src/context-virtualization/state.ts +79 -0
- package/pi-extension/src/context-virtualization/types.ts +92 -0
- package/pi-extension/src/conversation-history/passages.ts +81 -0
- package/pi-extension/src/conversation-history/ranking.ts +155 -0
- package/pi-extension/src/conversation-history/runtime.ts +250 -0
- package/pi-extension/src/conversation-history/search.ts +79 -0
- package/pi-extension/src/conversation-history/types.ts +31 -0
- package/pi-extension/src/freeflow-context/resolver.ts +139 -0
- package/pi-extension/src/freeflow-context/runtime.ts +100 -0
- package/pi-extension/src/freeflow-context/source-projector.ts +106 -0
- package/pi-extension/src/freeflow-context/tool.ts +360 -0
- package/pi-extension/src/freeflow-context/types.ts +70 -0
- package/pi-extension/src/freeflow-context/visibility.ts +53 -0
- package/pi-extension/src/index.ts +553 -101
- package/pi-extension/src/runtime/runtime-context.ts +339 -281
- package/pi-extension/src/runtime/runtime-identity.ts +8 -0
- package/pi-extension/src/settings/settings-tui.ts +93 -0
- package/pi-extension/src/settings/settings-ui.ts +700 -613
- package/pi-extension/src/ui/text-component.ts +66 -0
- package/pi-extension/tests/cognitive-routing/commands.test.js +134 -0
- package/pi-extension/tests/cognitive-routing/controller.test.js +1099 -0
- package/pi-extension/tests/cognitive-routing/extension-lifecycle.test.js +1071 -0
- package/pi-extension/tests/cognitive-routing/history.test.js +274 -0
- package/pi-extension/tests/cognitive-routing/host-fixture.js +4 -0
- package/pi-extension/tests/cognitive-routing/host-identity.test.js +63 -0
- package/pi-extension/tests/cognitive-routing/profile-resolution.test.js +236 -0
- package/pi-extension/tests/cognitive-routing/tool.test.js +230 -0
- package/pi-extension/tests/context-virtualization/extension.test.js +749 -0
- package/pi-extension/tests/context-virtualization/projector.test.js +110 -0
- package/pi-extension/tests/context-virtualization/runtime.test.js +216 -0
- package/pi-extension/tests/conversation-history/runtime.test.js +98 -0
- package/pi-extension/tests/conversation-history/search.test.js +97 -0
- package/pi-extension/tests/integration/pi-extension.test.js +927 -2048
- package/pi-extension/tests/integration/prompt-architecture.test.js +305 -0
- package/runtime/prompts/cognitive-routing.md +9 -0
- package/runtime/prompts/context-virtualization.md +3 -0
- package/runtime/prompts/conversation-history.md +3 -0
- package/runtime/prompts/core.md +13 -0
- package/runtime/prompts/interaction-contract.md +7 -0
- package/runtime/prompts/skills.md +65 -0
- package/skills/action-selection/SKILL.md +91 -0
- package/skills/bypass/SKILL.md +3 -2
- package/skills/commit-work/SKILL.md +1 -1
- package/skills/commit-work/references/staging-decisions.md +1 -1
- package/skills/decision-gate/SKILL.md +1 -1
- package/skills/design-for-depth/SKILL.md +14 -33
- package/skills/design-for-depth/references/design-pressure-signals.md +1 -1
- package/skills/design-for-depth/references/interface-design-loop.md +1 -1
- package/skills/design-for-depth/references/software-design-philosophy.md +1 -1
- package/skills/diagnose-failure/SKILL.md +20 -14
- package/skills/diagnose-failure/references/diagnostic-loop-catalog.md +1 -3
- package/skills/diagnose-failure/references/{flaky-and-performance.md → flaky-and-performance-diagnosis.md} +1 -1
- package/skills/discuss/SKILL.md +34 -22
- package/skills/discuss/references/discussion-continuity.md +71 -0
- package/skills/evaluate-skill/SKILL.md +5 -5
- package/skills/evaluate-skill/references/{definition-schema.md → evaluation-definition-schema.md} +42 -2
- package/skills/evaluate-skill/references/execution-and-evidence.md +8 -3
- package/skills/evaluate-skill/scripts/lib/definitions.mjs +114 -3
- package/skills/evaluate-skill/scripts/lib/grade-interaction.mjs +156 -1
- package/skills/evaluate-skill/scripts/lib/grade.mjs +9 -1
- package/skills/evaluate-skill/scripts/lib/pi.mjs +147 -32
- package/skills/evaluate-skill/scripts/lib/rpc.mjs +4 -4
- package/skills/evaluate-skill/scripts/lib/runner.mjs +36 -14
- package/skills/evaluate-skill/scripts/lib/sandbox.mjs +92 -0
- package/skills/evaluate-skill/scripts/pi-guard.mjs +1 -1
- package/skills/evaluate-skill/scripts/pi-observer.mjs +66 -0
- package/skills/evaluate-skill/scripts/skill-eval.mjs +6 -5
- package/skills/execute-work/SKILL.md +18 -15
- package/skills/execute-work/references/code-practices.md +1 -1
- package/skills/execute-work/references/execute-work-edges.md +42 -0
- package/skills/finish-branch/SKILL.md +2 -2
- package/skills/finish-branch/references/integration-options.md +1 -1
- package/skills/handoff/SKILL.md +3 -3
- package/skills/launch-work/SKILL.md +1 -1
- package/skills/launch-work/references/launch-readiness.md +2 -2
- package/skills/migration-work/SKILL.md +2 -2
- package/skills/migration-work/references/migration-lifecycle.md +1 -1
- package/skills/mode-contract/SKILL.md +4 -4
- package/skills/release-work/SKILL.md +1 -1
- package/skills/release-work/references/release-evidence.md +1 -1
- package/skills/review-artifact/SKILL.md +28 -26
- package/skills/review-work/SKILL.md +28 -26
- package/skills/review-work/references/{reviewer-prompt.md → independent-work-reviewer-contract.md} +1 -1
- package/skills/review-work/references/security-risk-lens.md +1 -1
- package/skills/setup-freeflow/SKILL.md +3 -1
- package/skills/setup-freeflow/references/activation-contract.md +2 -2
- package/skills/setup-freeflow/references/host-setup.md +2 -2
- package/skills/simplify-code/SKILL.md +1 -1
- package/skills/simplify-code/references/simplification-patterns.md +1 -1
- package/skills/tdd/SKILL.md +18 -38
- package/skills/tdd/references/test-design.md +2 -2
- package/skills/track-work/SKILL.md +362 -58
- package/skills/track-work/scripts/lib/codec.mjs +570 -0
- package/skills/track-work/scripts/lib/commands.mjs +568 -0
- package/skills/track-work/scripts/lib/contract.mjs +303 -0
- package/skills/track-work/scripts/lib/model.mjs +506 -0
- package/skills/track-work/scripts/lib/operations.mjs +965 -0
- package/skills/track-work/scripts/lib/record-store.mjs +264 -0
- package/skills/track-work/scripts/lib/result.mjs +37 -0
- package/skills/track-work/scripts/lib/rewrites.mjs +740 -0
- package/skills/track-work/scripts/lib/schema.mjs +277 -0
- package/skills/track-work/scripts/lib/source-inventory.mjs +162 -0
- package/skills/track-work/scripts/lib/views.mjs +443 -0
- package/skills/track-work/scripts/lib/workspace.mjs +134 -0
- package/skills/track-work/scripts/working-record.mjs +122 -0
- package/skills/verify-work/SKILL.md +14 -2
- package/skills/verify-work/references/browser-runtime-evidence.md +1 -1
- package/skills/verify-work/references/integration-evidence.md +1 -1
- package/skills/verify-work/references/performance-evidence.md +1 -1
- package/skills/workflow/SKILL.md +105 -53
- package/skills/workflow/references/domain-skill-composition.md +22 -42
- package/skills/write-plan/SKILL.md +1 -1
- package/skills/write-skill/SKILL.md +3 -2
- package/skills/write-skill/references/progressive-disclosure.md +17 -3
- package/capabilities/output-router/SKILL.md +0 -206
- package/capabilities/output-router/references/safety-policy.md +0 -40
- package/capabilities/output-router/references/setup.md +0 -110
- package/pi-extension/dist/output-router/host-producer-identification.js +0 -232
- package/pi-extension/dist/output-router/native-safety-net.js +0 -216
- package/pi-extension/dist/output-router/observed-tool-routing.js +0 -122
- package/pi-extension/dist/output-router/renderers.js +0 -1023
- package/pi-extension/dist/output-router/router-tools.js +0 -554
- package/pi-extension/dist/output-router/schemas.js +0 -383
- package/pi-extension/dist/output-router/status.js +0 -744
- package/pi-extension/dist/output-router/utils.js +0 -502
- package/pi-extension/src/output-router/host-producer-identification.ts +0 -256
- package/pi-extension/src/output-router/native-safety-net.ts +0 -248
- package/pi-extension/src/output-router/observed-tool-routing.ts +0 -134
- package/pi-extension/src/output-router/renderers.ts +0 -1108
- package/pi-extension/src/output-router/router-tools.ts +0 -579
- package/pi-extension/src/output-router/schemas.ts +0 -387
- package/pi-extension/src/output-router/status.ts +0 -791
- package/pi-extension/src/output-router/utils.ts +0 -554
- package/pi-extension/tests/output-router/observed-producers.test.js +0 -132
- package/pi-extension/tests/output-router/search.test.js +0 -722
- package/router/dist/benchmarks/benchmark-harness.d.ts +0 -35
- package/router/dist/benchmarks/benchmark-harness.js +0 -92
- package/router/dist/benchmarks/benchmarks.d.ts +0 -109
- package/router/dist/benchmarks/benchmarks.js +0 -979
- package/router/dist/benchmarks/codex-qa-benchmarks.d.ts +0 -95
- package/router/dist/benchmarks/codex-qa-benchmarks.js +0 -500
- package/router/dist/benchmarks/command-benchmarks.d.ts +0 -143
- package/router/dist/benchmarks/command-benchmarks.js +0 -698
- package/router/dist/benchmarks/context-mode-normalized-benchmarks.d.ts +0 -82
- package/router/dist/benchmarks/context-mode-normalized-benchmarks.js +0 -773
- package/router/dist/benchmarks/context-mode-real-deep-benchmark.d.ts +0 -132
- package/router/dist/benchmarks/context-mode-real-deep-benchmark.js +0 -1380
- package/router/dist/benchmarks/context-saving-baseline.d.ts +0 -60
- package/router/dist/benchmarks/context-saving-baseline.js +0 -470
- package/router/dist/benchmarks/index-benchmarks.d.ts +0 -103
- package/router/dist/benchmarks/index-benchmarks.js +0 -862
- package/router/dist/benchmarks/storage-policy-benchmarks.d.ts +0 -101
- package/router/dist/benchmarks/storage-policy-benchmarks.js +0 -618
- package/router/dist/benchmarks/vault-index-benchmarks.d.ts +0 -56
- package/router/dist/benchmarks/vault-index-benchmarks.js +0 -322
- package/router/dist/config/config.d.ts +0 -103
- package/router/dist/config/config.js +0 -632
- package/router/dist/config/router-contract.d.ts +0 -10
- package/router/dist/config/router-contract.js +0 -54
- package/router/dist/config/schema.d.ts +0 -27
- package/router/dist/config/schema.js +0 -648
- package/router/dist/config/types.d.ts +0 -573
- package/router/dist/config/types.js +0 -41
- package/router/dist/evidence/bounded-evidence.d.ts +0 -40
- package/router/dist/evidence/bounded-evidence.js +0 -323
- package/router/dist/evidence/evidence-range-selector.d.ts +0 -24
- package/router/dist/evidence/evidence-range-selector.js +0 -232
- package/router/dist/evidence/evidence-search.d.ts +0 -27
- package/router/dist/evidence/evidence-search.js +0 -490
- package/router/dist/evidence/evidence.d.ts +0 -38
- package/router/dist/evidence/evidence.js +0 -200
- package/router/dist/evidence/failure-contracts.d.ts +0 -37
- package/router/dist/evidence/failure-contracts.js +0 -166
- package/router/dist/evidence/line-ranges.d.ts +0 -24
- package/router/dist/evidence/line-ranges.js +0 -22
- package/router/dist/experiments/local-index.d.ts +0 -64
- package/router/dist/experiments/local-index.js +0 -581
- package/router/dist/index.d.ts +0 -30
- package/router/dist/index.js +0 -32
- package/router/dist/local/local-traversal.d.ts +0 -18
- package/router/dist/local/local-traversal.js +0 -334
- package/router/dist/processing/engine.d.ts +0 -132
- package/router/dist/processing/engine.js +0 -533
- package/router/dist/processing/reducers.d.ts +0 -280
- package/router/dist/processing/reducers.js +0 -1673
- package/router/dist/processing/renderers.d.ts +0 -31
- package/router/dist/processing/renderers.js +0 -195
- package/router/dist/processing/scripts.d.ts +0 -91
- package/router/dist/processing/scripts.js +0 -386
- package/router/dist/repo/repo-traversal.d.ts +0 -18
- package/router/dist/repo/repo-traversal.js +0 -245
- package/router/dist/routing/observed-reducers.d.ts +0 -41
- package/router/dist/routing/observed-reducers.js +0 -661
- package/router/dist/routing/observed-routing.d.ts +0 -26
- package/router/dist/routing/observed-routing.js +0 -378
- package/router/dist/routing/parsers.d.ts +0 -16
- package/router/dist/routing/parsers.js +0 -521
- package/router/dist/routing/run-filters.d.ts +0 -60
- package/router/dist/routing/run-filters.js +0 -238
- package/router/dist/routing/run-reducers.d.ts +0 -49
- package/router/dist/routing/run-reducers.js +0 -122
- package/router/dist/sandbox/adapter-roots.d.ts +0 -46
- package/router/dist/sandbox/adapter-roots.js +0 -66
- package/router/dist/sandbox/eryx-python-adapter.d.ts +0 -13
- package/router/dist/sandbox/eryx-python-adapter.js +0 -868
- package/router/dist/sandbox/jq-wasm-adapter.d.ts +0 -9
- package/router/dist/sandbox/jq-wasm-adapter.js +0 -429
- package/router/dist/sandbox/quickjs-wasi-adapter.d.ts +0 -13
- package/router/dist/sandbox/quickjs-wasi-adapter.js +0 -384
- package/router/dist/sandbox/script-sandbox.d.ts +0 -134
- package/router/dist/sandbox/script-sandbox.js +0 -337
- package/router/dist/setup/script-transform-adapters.d.ts +0 -32
- package/router/dist/setup/script-transform-adapters.js +0 -278
- package/router/dist/tools/batch.d.ts +0 -33
- package/router/dist/tools/batch.js +0 -586
- package/router/dist/tools/run.d.ts +0 -62
- package/router/dist/tools/run.js +0 -1304
- package/router/dist/tools/search.d.ts +0 -66
- package/router/dist/tools/search.js +0 -1364
- package/router/dist/transform/engine.d.ts +0 -168
- package/router/dist/transform/engine.js +0 -2072
- package/router/dist/vault/vault-index.d.ts +0 -125
- package/router/dist/vault/vault-index.js +0 -416
- package/router/dist/vault/vault.d.ts +0 -138
- package/router/dist/vault/vault.js +0 -630
- package/runtime/interaction-contract.md +0 -16
- package/skills/discuss/references/checkpoints.md +0 -66
- package/skills/execute-work/references/execution-loop.md +0 -77
- package/skills/track-work/references/working-record-schema.md +0 -378
- package/skills/workflow/references/workflow-loop.md +0 -92
- /package/skills/handoff/references/{templates.md → handoff-templates.md} +0 -0
- /package/skills/review-artifact/references/{reviewer-prompt.md → independent-artifact-reviewer-contract.md} +0 -0
|
@@ -0,0 +1,356 @@
|
|
|
1
|
+
import {
|
|
2
|
+
CONTEXT_PROJECTION_ENTRY,
|
|
3
|
+
CONTEXT_PROJECTION_VERSION,
|
|
4
|
+
contextRefForEntry,
|
|
5
|
+
entryIdFromContextRef,
|
|
6
|
+
} from "./types.js";
|
|
7
|
+
import { FreeflowContextRuntime } from "../freeflow-context/runtime.js";
|
|
8
|
+
import { contentCharacters, projectToolResultMessage, projectedCharacters } from "./projector.js";
|
|
9
|
+
import { replayProjectionEntries } from "./state.js";
|
|
10
|
+
function cloneSource(source) {
|
|
11
|
+
return { ...source };
|
|
12
|
+
}
|
|
13
|
+
function sourceProjection(state) {
|
|
14
|
+
if (!state || state.mode === "full") return { mode: "full" };
|
|
15
|
+
return state.retained === undefined ? { mode: "archived" } : { mode: "archived", retained: state.retained };
|
|
16
|
+
}
|
|
17
|
+
function safeJsonEqual(left, right) {
|
|
18
|
+
try {
|
|
19
|
+
return JSON.stringify(left) === JSON.stringify(right);
|
|
20
|
+
} catch {
|
|
21
|
+
return false;
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
function isDuplicate(values) {
|
|
25
|
+
const seen = new Set();
|
|
26
|
+
for (const value of values) {
|
|
27
|
+
if (seen.has(value)) return value;
|
|
28
|
+
seen.add(value);
|
|
29
|
+
}
|
|
30
|
+
return undefined;
|
|
31
|
+
}
|
|
32
|
+
export class ContextVirtualizationRuntime {
|
|
33
|
+
pi;
|
|
34
|
+
freeflowContext;
|
|
35
|
+
ctx;
|
|
36
|
+
sessionId = "current";
|
|
37
|
+
branchLeafId;
|
|
38
|
+
projectionState = new Map();
|
|
39
|
+
available = true;
|
|
40
|
+
unavailableReason;
|
|
41
|
+
mutationQueue = Promise.resolve();
|
|
42
|
+
constructor(pi, ctx, freeflowContext = new FreeflowContextRuntime(ctx)) {
|
|
43
|
+
this.pi = pi;
|
|
44
|
+
this.ctx = ctx;
|
|
45
|
+
this.freeflowContext = freeflowContext;
|
|
46
|
+
}
|
|
47
|
+
setContext(ctx) {
|
|
48
|
+
this.ctx = ctx;
|
|
49
|
+
this.freeflowContext.setContext(ctx);
|
|
50
|
+
}
|
|
51
|
+
async recover(ctx = this.ctx, options = {}) {
|
|
52
|
+
this.ctx = ctx;
|
|
53
|
+
this.freeflowContext.setContext(ctx);
|
|
54
|
+
try {
|
|
55
|
+
this.sessionId = this.freeflowContext.sessionId();
|
|
56
|
+
this.branchLeafId = this.freeflowContext.branchLeafId();
|
|
57
|
+
this.projectionState = replayProjectionEntries(this.freeflowContext.branchEntries(), this.sessionId);
|
|
58
|
+
if (!options.preserveRequest) this.freeflowContext.clearRequest();
|
|
59
|
+
this.available = true;
|
|
60
|
+
this.unavailableReason = undefined;
|
|
61
|
+
return true;
|
|
62
|
+
} catch (error) {
|
|
63
|
+
this.available = false;
|
|
64
|
+
this.unavailableReason = error instanceof Error ? error.message : String(error);
|
|
65
|
+
this.freeflowContext.clearRequest();
|
|
66
|
+
return false;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
isAvailable() {
|
|
70
|
+
return this.available;
|
|
71
|
+
}
|
|
72
|
+
isSourceFullyProjected(entryId) {
|
|
73
|
+
if (!this.available) return true;
|
|
74
|
+
const projection = this.projectionState.get(entryId);
|
|
75
|
+
return !projection || projection.mode === "full";
|
|
76
|
+
}
|
|
77
|
+
async project(messages, enabled) {
|
|
78
|
+
if (!enabled) {
|
|
79
|
+
this.freeflowContext.clearRequest();
|
|
80
|
+
return { messages, changed: false, available: true };
|
|
81
|
+
}
|
|
82
|
+
if (!(await this.recover(this.ctx))) {
|
|
83
|
+
return { messages, changed: false, available: false };
|
|
84
|
+
}
|
|
85
|
+
const sourcesByToolCallId = this.freeflowContext.resolver.toolResultSources();
|
|
86
|
+
const refs = new Map();
|
|
87
|
+
let changed = false;
|
|
88
|
+
const projected = messages.map((message) => {
|
|
89
|
+
if (message?.role !== "toolResult" || typeof message.toolCallId !== "string") return message;
|
|
90
|
+
const source = sourcesByToolCallId.get(message.toolCallId);
|
|
91
|
+
if (!source) return message;
|
|
92
|
+
const ref = contextRefForEntry(source.source.entryId);
|
|
93
|
+
refs.set(ref, source);
|
|
94
|
+
const state = this.projectionState.get(source.source.entryId);
|
|
95
|
+
const projection = sourceProjection(state);
|
|
96
|
+
const nextMessage = projectToolResultMessage(message, source.source, projection);
|
|
97
|
+
if (!safeJsonEqual(nextMessage.content, message.content)) changed = true;
|
|
98
|
+
return nextMessage;
|
|
99
|
+
});
|
|
100
|
+
const request = this.freeflowContext.recordRequest(refs);
|
|
101
|
+
return {
|
|
102
|
+
messages: changed ? projected : messages,
|
|
103
|
+
changed,
|
|
104
|
+
available: true,
|
|
105
|
+
generation: request.generation,
|
|
106
|
+
};
|
|
107
|
+
}
|
|
108
|
+
async archive(targets) {
|
|
109
|
+
try {
|
|
110
|
+
return await this.enqueue(() => this.archiveNow(targets));
|
|
111
|
+
} catch (error) {
|
|
112
|
+
return this.rejected("archive", `persistence_failed:${error instanceof Error ? error.message : String(error)}`);
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
async restore(refs, actor = "model") {
|
|
116
|
+
try {
|
|
117
|
+
return await this.enqueue(() => this.restoreNow(refs, actor));
|
|
118
|
+
} catch (error) {
|
|
119
|
+
return this.rejected("restore", `persistence_failed:${error instanceof Error ? error.message : String(error)}`);
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
async reset() {
|
|
123
|
+
try {
|
|
124
|
+
return await this.enqueue(async () => {
|
|
125
|
+
if (!this.available) return this.unavailable("reset");
|
|
126
|
+
const journal = {
|
|
127
|
+
version: CONTEXT_PROJECTION_VERSION,
|
|
128
|
+
actor: "user",
|
|
129
|
+
reset: "all",
|
|
130
|
+
};
|
|
131
|
+
await this.persist(journal);
|
|
132
|
+
this.projectionState.clear();
|
|
133
|
+
return {
|
|
134
|
+
status: "ok",
|
|
135
|
+
operation: "reset",
|
|
136
|
+
changed: ["all"],
|
|
137
|
+
message: "All context projection decisions were reset on the active branch.",
|
|
138
|
+
};
|
|
139
|
+
});
|
|
140
|
+
} catch (error) {
|
|
141
|
+
return this.rejected("reset", `persistence_failed:${error instanceof Error ? error.message : String(error)}`);
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
async status(limit = 50) {
|
|
145
|
+
if (!this.available) {
|
|
146
|
+
return {
|
|
147
|
+
available: false,
|
|
148
|
+
unavailableReason: this.unavailableReason,
|
|
149
|
+
sessionId: this.sessionId,
|
|
150
|
+
branchLeafId: this.branchLeafId,
|
|
151
|
+
counts: { full: 0, archived: 0, retained: 0 },
|
|
152
|
+
originalCharacters: 0,
|
|
153
|
+
projectedCharacters: 0,
|
|
154
|
+
historyOnly: 0,
|
|
155
|
+
unresolved: 0,
|
|
156
|
+
items: [],
|
|
157
|
+
};
|
|
158
|
+
}
|
|
159
|
+
if (!(await this.recover(this.ctx, { preserveRequest: true }))) {
|
|
160
|
+
return this.status(limit);
|
|
161
|
+
}
|
|
162
|
+
const branchSources = this.freeflowContext.resolver.sourcesByEntryId();
|
|
163
|
+
const activeIds = this.freeflowContext.resolver.activeToolResultIds();
|
|
164
|
+
const items = [];
|
|
165
|
+
for (const source of branchSources.values()) {
|
|
166
|
+
const state = this.projectionState.get(source.source.entryId);
|
|
167
|
+
const projection = sourceProjection(state);
|
|
168
|
+
items.push(this.listItem(source, projection, activeIds.has(source.source.entryId) ? "active" : "history-only"));
|
|
169
|
+
}
|
|
170
|
+
for (const state of this.projectionState.values()) {
|
|
171
|
+
if (branchSources.has(state.source.entryId)) continue;
|
|
172
|
+
const projection = sourceProjection(state);
|
|
173
|
+
items.push({
|
|
174
|
+
ref: contextRefForEntry(state.source.entryId),
|
|
175
|
+
entryId: state.source.entryId,
|
|
176
|
+
toolCallId: state.source.toolCallId,
|
|
177
|
+
toolName: state.source.toolName,
|
|
178
|
+
mode: projection.mode,
|
|
179
|
+
...(projection.mode === "archived" && projection.retained !== undefined
|
|
180
|
+
? { retained: projection.retained }
|
|
181
|
+
: {}),
|
|
182
|
+
originalCharacters: 0,
|
|
183
|
+
projectedCharacters: projection.mode === "archived" ? contentCharacters(projection.retained ?? "") : 0,
|
|
184
|
+
availability: "unresolved",
|
|
185
|
+
});
|
|
186
|
+
}
|
|
187
|
+
const counts = { full: 0, archived: 0, retained: 0 };
|
|
188
|
+
let originalCharacters = 0;
|
|
189
|
+
let projectedCharactersTotal = 0;
|
|
190
|
+
let historyOnly = 0;
|
|
191
|
+
let unresolved = 0;
|
|
192
|
+
for (const item of items) {
|
|
193
|
+
if (item.mode === "full") counts.full += 1;
|
|
194
|
+
else {
|
|
195
|
+
counts.archived += 1;
|
|
196
|
+
if (item.retained !== undefined) counts.retained += 1;
|
|
197
|
+
}
|
|
198
|
+
originalCharacters += item.originalCharacters;
|
|
199
|
+
projectedCharactersTotal += item.projectedCharacters;
|
|
200
|
+
if (item.availability === "history-only") historyOnly += 1;
|
|
201
|
+
if (item.availability === "unresolved") unresolved += 1;
|
|
202
|
+
}
|
|
203
|
+
return {
|
|
204
|
+
available: true,
|
|
205
|
+
sessionId: this.sessionId,
|
|
206
|
+
branchLeafId: this.branchLeafId,
|
|
207
|
+
counts,
|
|
208
|
+
originalCharacters,
|
|
209
|
+
projectedCharacters: projectedCharactersTotal,
|
|
210
|
+
historyOnly,
|
|
211
|
+
unresolved,
|
|
212
|
+
items: items.slice(-Math.max(1, limit)),
|
|
213
|
+
};
|
|
214
|
+
}
|
|
215
|
+
async archiveNow(targets) {
|
|
216
|
+
if (!this.available) return this.unavailable("archive");
|
|
217
|
+
if (!Array.isArray(targets) || targets.length === 0) {
|
|
218
|
+
return this.rejected("archive", "targets_required");
|
|
219
|
+
}
|
|
220
|
+
const normalizedRefs = targets.map((target) =>
|
|
221
|
+
target && typeof target === "object" ? entryIdFromContextRef(target.ref) : undefined,
|
|
222
|
+
);
|
|
223
|
+
const duplicate = isDuplicate(
|
|
224
|
+
normalizedRefs.flatMap((ref) => (ref === undefined ? [] : [contextRefForEntry(ref)])),
|
|
225
|
+
);
|
|
226
|
+
if (duplicate) return this.rejected("archive", `duplicate_reference:${duplicate}`);
|
|
227
|
+
const latestRequest = this.freeflowContext.latestRequest();
|
|
228
|
+
if (!latestRequest) return this.rejected("archive", "no_consumed_context_request");
|
|
229
|
+
const changes = [];
|
|
230
|
+
const retainedMeaning = {};
|
|
231
|
+
for (let index = 0; index < targets.length; index += 1) {
|
|
232
|
+
const target = targets[index];
|
|
233
|
+
if (!target || typeof target !== "object") return this.rejected("archive", `target_${index}_must_be_object`);
|
|
234
|
+
const ref = normalizedRefs[index];
|
|
235
|
+
if (!ref) return this.rejected("archive", `target_${index}_invalid_reference`);
|
|
236
|
+
const normalizedRef = contextRefForEntry(ref);
|
|
237
|
+
const source = latestRequest.refs.get(normalizedRef);
|
|
238
|
+
if (!source) return this.rejected("archive", `target_not_in_consumed_context:${normalizedRef}`);
|
|
239
|
+
const retainedValue = target.retained;
|
|
240
|
+
if (retainedValue !== undefined && (typeof retainedValue !== "string" || retainedValue.trim() === "")) {
|
|
241
|
+
return this.rejected("archive", `target_${index}_retained_must_be_non_empty`);
|
|
242
|
+
}
|
|
243
|
+
const retained = typeof retainedValue === "string" ? retainedValue.trim() : undefined;
|
|
244
|
+
changes.push({
|
|
245
|
+
source: cloneSource(source.source),
|
|
246
|
+
projection: retained === undefined ? { mode: "archived" } : { mode: "archived", retained },
|
|
247
|
+
});
|
|
248
|
+
if (retained !== undefined) retainedMeaning[normalizedRef] = retained;
|
|
249
|
+
}
|
|
250
|
+
await this.persist({ version: CONTEXT_PROJECTION_VERSION, actor: "model", changes });
|
|
251
|
+
for (const change of changes) {
|
|
252
|
+
this.projectionState.set(change.source.entryId, {
|
|
253
|
+
source: cloneSource(change.source),
|
|
254
|
+
...change.projection,
|
|
255
|
+
});
|
|
256
|
+
}
|
|
257
|
+
return {
|
|
258
|
+
status: "ok",
|
|
259
|
+
operation: "archive",
|
|
260
|
+
changed: changes.map((change) => contextRefForEntry(change.source.entryId)),
|
|
261
|
+
message: `Archived ${changes.length} tool result${changes.length === 1 ? "" : "s"} from future context projections.`,
|
|
262
|
+
...(Object.keys(retainedMeaning).length > 0 ? { retained: retainedMeaning } : {}),
|
|
263
|
+
};
|
|
264
|
+
}
|
|
265
|
+
async restoreNow(refs, actor) {
|
|
266
|
+
if (!this.available) return this.unavailable("restore");
|
|
267
|
+
if (!Array.isArray(refs) || refs.length === 0) return this.rejected("restore", "refs_required");
|
|
268
|
+
const normalizedRefs = refs.map((ref) => entryIdFromContextRef(ref));
|
|
269
|
+
const duplicate = isDuplicate(
|
|
270
|
+
normalizedRefs.flatMap((ref) => (ref === undefined ? [] : [contextRefForEntry(ref)])),
|
|
271
|
+
);
|
|
272
|
+
if (duplicate) return this.rejected("restore", `duplicate_reference:${duplicate}`);
|
|
273
|
+
const changes = [];
|
|
274
|
+
const availability = {};
|
|
275
|
+
const activeIds = this.freeflowContext.resolver.activeToolResultIds();
|
|
276
|
+
const latestRequest = this.freeflowContext.latestRequest();
|
|
277
|
+
for (let index = 0; index < refs.length; index += 1) {
|
|
278
|
+
const ref = normalizedRefs[index];
|
|
279
|
+
if (!ref) return this.rejected("restore", `ref_${index}_invalid_reference`);
|
|
280
|
+
const normalizedRef = contextRefForEntry(ref);
|
|
281
|
+
const current = this.projectionState.get(ref);
|
|
282
|
+
if (!current) return this.rejected("restore", `reference_not_archived:${normalizedRef}`);
|
|
283
|
+
if (actor === "model" && !latestRequest?.refs.has(normalizedRef)) {
|
|
284
|
+
return this.rejected("restore", `target_not_in_consumed_context:${normalizedRef}`);
|
|
285
|
+
}
|
|
286
|
+
changes.push({ source: cloneSource(current.source), projection: { mode: "full" } });
|
|
287
|
+
let availabilityValue;
|
|
288
|
+
if (latestRequest?.refs.has(normalizedRef) || activeIds.has(current.source.entryId)) {
|
|
289
|
+
availabilityValue = "active";
|
|
290
|
+
} else {
|
|
291
|
+
availabilityValue = "history-only";
|
|
292
|
+
}
|
|
293
|
+
availability[normalizedRef] = availabilityValue;
|
|
294
|
+
}
|
|
295
|
+
await this.persist({ version: CONTEXT_PROJECTION_VERSION, actor, changes });
|
|
296
|
+
for (const change of changes) {
|
|
297
|
+
this.projectionState.set(change.source.entryId, {
|
|
298
|
+
source: cloneSource(change.source),
|
|
299
|
+
mode: "full",
|
|
300
|
+
});
|
|
301
|
+
}
|
|
302
|
+
return {
|
|
303
|
+
status: "ok",
|
|
304
|
+
operation: "restore",
|
|
305
|
+
changed: changes.map((change) => contextRefForEntry(change.source.entryId)),
|
|
306
|
+
availability,
|
|
307
|
+
message: `Restored ${changes.length} projection${changes.length === 1 ? "" : "s"} to full content.`,
|
|
308
|
+
};
|
|
309
|
+
}
|
|
310
|
+
listItem(source, projection, availability) {
|
|
311
|
+
const item = {
|
|
312
|
+
ref: contextRefForEntry(source.source.entryId),
|
|
313
|
+
entryId: source.source.entryId,
|
|
314
|
+
toolCallId: source.source.toolCallId,
|
|
315
|
+
toolName: source.source.toolName,
|
|
316
|
+
mode: projection.mode,
|
|
317
|
+
originalCharacters: contentCharacters(source.message?.content),
|
|
318
|
+
projectedCharacters: projectedCharacters(source.source, source.message, projection),
|
|
319
|
+
availability,
|
|
320
|
+
};
|
|
321
|
+
if (projection.mode === "archived" && projection.retained !== undefined) {
|
|
322
|
+
item.retained = projection.retained;
|
|
323
|
+
}
|
|
324
|
+
return item;
|
|
325
|
+
}
|
|
326
|
+
rejected(operation, message) {
|
|
327
|
+
return { status: "rejected", operation, changed: [], message };
|
|
328
|
+
}
|
|
329
|
+
unavailable(operation) {
|
|
330
|
+
return {
|
|
331
|
+
status: "unavailable",
|
|
332
|
+
operation,
|
|
333
|
+
changed: [],
|
|
334
|
+
message: this.unavailableReason ?? "context_virtualization_unavailable",
|
|
335
|
+
};
|
|
336
|
+
}
|
|
337
|
+
persist(journal) {
|
|
338
|
+
let append;
|
|
339
|
+
if (typeof this.pi?.appendEntryDurable === "function") {
|
|
340
|
+
append = this.pi.appendEntryDurable;
|
|
341
|
+
} else if (typeof this.pi?.appendEntry === "function") {
|
|
342
|
+
append = this.pi.appendEntry;
|
|
343
|
+
}
|
|
344
|
+
if (!append) throw new Error("The Pi host does not expose a session entry persistence method.");
|
|
345
|
+
const result = append.call(this.pi, CONTEXT_PROJECTION_ENTRY, journal);
|
|
346
|
+
if (result && typeof result.then === "function") return result;
|
|
347
|
+
}
|
|
348
|
+
enqueue(task) {
|
|
349
|
+
const run = this.mutationQueue.then(task);
|
|
350
|
+
this.mutationQueue = run.then(
|
|
351
|
+
() => undefined,
|
|
352
|
+
() => undefined,
|
|
353
|
+
);
|
|
354
|
+
return run;
|
|
355
|
+
}
|
|
356
|
+
}
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import { CONTEXT_PROJECTION_ENTRY, CONTEXT_PROJECTION_VERSION, isRecord } from "./types.js";
|
|
2
|
+
function isProjection(value) {
|
|
3
|
+
if (!isRecord(value) || (value.mode !== "full" && value.mode !== "archived")) return false;
|
|
4
|
+
if (value.mode === "archived" && value.retained !== undefined && typeof value.retained !== "string") return false;
|
|
5
|
+
return true;
|
|
6
|
+
}
|
|
7
|
+
function isJournal(value) {
|
|
8
|
+
if (!isRecord(value) || value.version !== CONTEXT_PROJECTION_VERSION) return false;
|
|
9
|
+
if (value.actor !== "model" && value.actor !== "user") return false;
|
|
10
|
+
if (value.reset !== undefined && value.reset !== "all") return false;
|
|
11
|
+
if (value.changes !== undefined && !Array.isArray(value.changes)) return false;
|
|
12
|
+
return true;
|
|
13
|
+
}
|
|
14
|
+
export function replayProjectionEntries(entries, sessionId) {
|
|
15
|
+
const state = new Map();
|
|
16
|
+
for (const entry of entries) {
|
|
17
|
+
if (entry?.type !== "custom" || entry.customType !== CONTEXT_PROJECTION_ENTRY) continue;
|
|
18
|
+
const data = entry.data;
|
|
19
|
+
if (!isJournal(data)) continue;
|
|
20
|
+
if (data.reset === "all") {
|
|
21
|
+
state.clear();
|
|
22
|
+
}
|
|
23
|
+
for (const change of data.changes ?? []) {
|
|
24
|
+
if (!isRecord(change) || !isRecord(change.source) || !isProjection(change.projection)) continue;
|
|
25
|
+
if (change.source.sessionId !== sessionId || typeof change.source.entryId !== "string") continue;
|
|
26
|
+
if (change.projection.mode === "archived" && change.projection.retained !== undefined) {
|
|
27
|
+
state.set(change.source.entryId, {
|
|
28
|
+
source: { ...change.source },
|
|
29
|
+
mode: "archived",
|
|
30
|
+
retained: change.projection.retained,
|
|
31
|
+
});
|
|
32
|
+
} else if (change.projection.mode === "archived") {
|
|
33
|
+
state.set(change.source.entryId, {
|
|
34
|
+
source: { ...change.source },
|
|
35
|
+
mode: "archived",
|
|
36
|
+
});
|
|
37
|
+
} else {
|
|
38
|
+
state.set(change.source.entryId, {
|
|
39
|
+
source: { ...change.source },
|
|
40
|
+
mode: "full",
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
return state;
|
|
46
|
+
}
|
|
47
|
+
export function cloneProjectionState(state) {
|
|
48
|
+
return new Map(
|
|
49
|
+
[...state.entries()].map(([entryId, projection]) => [
|
|
50
|
+
entryId,
|
|
51
|
+
{
|
|
52
|
+
...projection,
|
|
53
|
+
source: { ...projection.source },
|
|
54
|
+
},
|
|
55
|
+
]),
|
|
56
|
+
);
|
|
57
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
export { CONTEXT_REF_PREFIX, contextRefForEntry, entryIdFromContextRef } from "../freeflow-context/types.js";
|
|
2
|
+
export const CONTEXT_PROJECTION_ENTRY = "freeflow-context-projection";
|
|
3
|
+
export const CONTEXT_PROJECTION_VERSION = 1;
|
|
4
|
+
export function isRecord(value) {
|
|
5
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
6
|
+
}
|
|
7
|
+
export function projectionCharacters(projection) {
|
|
8
|
+
if (projection.mode === "full") return 0;
|
|
9
|
+
return projection.retained?.length ?? 0;
|
|
10
|
+
}
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
export const PASSAGE_TARGET_CHARACTERS = 2_000;
|
|
2
|
+
export const PASSAGE_OVERLAP_CHARACTERS = 200;
|
|
3
|
+
export const SNIPPET_MAX_CHARACTERS = 400;
|
|
4
|
+
export function normalizeText(value) {
|
|
5
|
+
return value.normalize("NFKC").toLocaleLowerCase().replace(/\s+/g, " ").trim();
|
|
6
|
+
}
|
|
7
|
+
export function tokenize(value) {
|
|
8
|
+
return normalizeText(value).match(/[\p{L}\p{N}_]+(?:[./:@#$%+-][\p{L}\p{N}_]+)*/gu) ?? [];
|
|
9
|
+
}
|
|
10
|
+
export function unique(values) {
|
|
11
|
+
return [...new Set(values)];
|
|
12
|
+
}
|
|
13
|
+
export function throwIfAborted(signal) {
|
|
14
|
+
if (!signal?.aborted) return;
|
|
15
|
+
const error = new Error("conversation_history_search_cancelled");
|
|
16
|
+
error.name = "AbortError";
|
|
17
|
+
throw error;
|
|
18
|
+
}
|
|
19
|
+
function passageEnd(text, start) {
|
|
20
|
+
const target = Math.min(text.length, start + PASSAGE_TARGET_CHARACTERS);
|
|
21
|
+
if (target === text.length) return target;
|
|
22
|
+
const paragraphBreak = text.lastIndexOf("\n\n", target);
|
|
23
|
+
if (paragraphBreak > start + PASSAGE_TARGET_CHARACTERS / 2) return paragraphBreak + 2;
|
|
24
|
+
const lineBreak = text.lastIndexOf("\n", target);
|
|
25
|
+
if (lineBreak > start + PASSAGE_TARGET_CHARACTERS / 2) return lineBreak + 1;
|
|
26
|
+
return target;
|
|
27
|
+
}
|
|
28
|
+
export function splitTextIntoPassages(text, signal) {
|
|
29
|
+
throwIfAborted(signal);
|
|
30
|
+
if (text.length <= PASSAGE_TARGET_CHARACTERS) {
|
|
31
|
+
return [{ text, start: 0, end: text.length, tokens: tokenize(text), length: Math.max(1, tokenize(text).length) }];
|
|
32
|
+
}
|
|
33
|
+
const result = [];
|
|
34
|
+
let start = 0;
|
|
35
|
+
while (start < text.length) {
|
|
36
|
+
throwIfAborted(signal);
|
|
37
|
+
const end = passageEnd(text, start);
|
|
38
|
+
const passageText = text.slice(start, end);
|
|
39
|
+
const tokens = tokenize(passageText);
|
|
40
|
+
result.push({ text: passageText, start, end, tokens, length: Math.max(1, tokens.length) });
|
|
41
|
+
if (end >= text.length) break;
|
|
42
|
+
start = Math.max(start + 1, end - PASSAGE_OVERLAP_CHARACTERS);
|
|
43
|
+
}
|
|
44
|
+
return result;
|
|
45
|
+
}
|
|
46
|
+
export function snippet(text, phrase, terms) {
|
|
47
|
+
if (text.length <= SNIPPET_MAX_CHARACTERS) return text;
|
|
48
|
+
const lower = text.toLocaleLowerCase();
|
|
49
|
+
const phraseIndex = phrase ? lower.indexOf(phrase) : -1;
|
|
50
|
+
const termIndex =
|
|
51
|
+
phraseIndex >= 0
|
|
52
|
+
? phraseIndex
|
|
53
|
+
: terms.reduce((best, term) => {
|
|
54
|
+
const index = lower.indexOf(term);
|
|
55
|
+
return index >= 0 && (best < 0 || index < best) ? index : best;
|
|
56
|
+
}, -1);
|
|
57
|
+
const center = termIndex >= 0 ? termIndex : 0;
|
|
58
|
+
let start = Math.max(0, center - Math.floor(SNIPPET_MAX_CHARACTERS / 3));
|
|
59
|
+
const end = Math.min(text.length, start + SNIPPET_MAX_CHARACTERS);
|
|
60
|
+
if (end - start < SNIPPET_MAX_CHARACTERS) start = Math.max(0, end - SNIPPET_MAX_CHARACTERS);
|
|
61
|
+
const prefix = start > 0 ? "…" : "";
|
|
62
|
+
const suffix = end < text.length ? "…" : "";
|
|
63
|
+
const available = SNIPPET_MAX_CHARACTERS - prefix.length - suffix.length;
|
|
64
|
+
const body = text.slice(start, start + available);
|
|
65
|
+
return `${prefix}${body}${suffix}`;
|
|
66
|
+
}
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
import { normalizeText, splitTextIntoPassages, throwIfAborted, tokenize, unique } from "./passages.js";
|
|
2
|
+
export const BM25_K1 = 1.2;
|
|
3
|
+
export const BM25_B = 0.75;
|
|
4
|
+
function bm25Score(tokens, terms, documentFrequency, documentCount, averageLength) {
|
|
5
|
+
const counts = new Map();
|
|
6
|
+
for (const token of tokens) counts.set(token, (counts.get(token) ?? 0) + 1);
|
|
7
|
+
const length = Math.max(1, tokens.length);
|
|
8
|
+
let score = 0;
|
|
9
|
+
for (const term of terms) {
|
|
10
|
+
const termFrequency = counts.get(term) ?? 0;
|
|
11
|
+
if (termFrequency === 0) continue;
|
|
12
|
+
const df = documentFrequency.get(term) ?? 0;
|
|
13
|
+
const inverseDocumentFrequency = Math.log(1 + (documentCount - df + 0.5) / (df + 0.5));
|
|
14
|
+
const normalization = BM25_K1 * (1 - BM25_B + BM25_B * (length / Math.max(1, averageLength)));
|
|
15
|
+
score += inverseDocumentFrequency * ((termFrequency * (BM25_K1 + 1)) / (termFrequency + normalization));
|
|
16
|
+
}
|
|
17
|
+
return score;
|
|
18
|
+
}
|
|
19
|
+
function comparePassages(left, right) {
|
|
20
|
+
if (left.exactPhrase !== right.exactPhrase) return left.exactPhrase ? -1 : 1;
|
|
21
|
+
if (left.score !== right.score) return right.score - left.score;
|
|
22
|
+
if (left.matchedTerms.length !== right.matchedTerms.length) {
|
|
23
|
+
return right.matchedTerms.length - left.matchedTerms.length;
|
|
24
|
+
}
|
|
25
|
+
if (left.entry.position !== right.entry.position) return right.entry.position - left.entry.position;
|
|
26
|
+
return left.passage.start - right.passage.start;
|
|
27
|
+
}
|
|
28
|
+
function compareTextPassages(left, right) {
|
|
29
|
+
if (left.exactPhrase !== right.exactPhrase) return left.exactPhrase ? -1 : 1;
|
|
30
|
+
if (left.score !== right.score) return right.score - left.score;
|
|
31
|
+
if (left.matchedTerms.length !== right.matchedTerms.length) {
|
|
32
|
+
return right.matchedTerms.length - left.matchedTerms.length;
|
|
33
|
+
}
|
|
34
|
+
return left.passage.start - right.passage.start;
|
|
35
|
+
}
|
|
36
|
+
function rankInputs(inputs, terms, signal) {
|
|
37
|
+
const documentFrequency = new Map();
|
|
38
|
+
for (const input of inputs) {
|
|
39
|
+
throwIfAborted(signal);
|
|
40
|
+
for (const term of unique(input.passage.tokens)) {
|
|
41
|
+
documentFrequency.set(term, (documentFrequency.get(term) ?? 0) + 1);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
const averageLength =
|
|
45
|
+
inputs.length === 0 ? 1 : inputs.reduce((total, input) => total + input.passage.length, 0) / inputs.length;
|
|
46
|
+
const matching = [];
|
|
47
|
+
for (const input of inputs) {
|
|
48
|
+
throwIfAborted(signal);
|
|
49
|
+
if (input.matchedTerms.length > 0 || input.exactPhrase) matching.push(input);
|
|
50
|
+
}
|
|
51
|
+
return matching
|
|
52
|
+
.map((input) => ({
|
|
53
|
+
...input,
|
|
54
|
+
score: bm25Score(input.passage.tokens, terms, documentFrequency, inputs.length, averageLength),
|
|
55
|
+
}))
|
|
56
|
+
.sort((left, right) => {
|
|
57
|
+
if (left.entry && right.entry) {
|
|
58
|
+
return comparePassages({ ...left, entry: left.entry }, { ...right, entry: right.entry });
|
|
59
|
+
}
|
|
60
|
+
return compareTextPassages(left, right);
|
|
61
|
+
});
|
|
62
|
+
}
|
|
63
|
+
export function rankMatchingPassages(entries, query, signal) {
|
|
64
|
+
throwIfAborted(signal);
|
|
65
|
+
const normalizedQuery = normalizeText(query);
|
|
66
|
+
const terms = unique(tokenize(query));
|
|
67
|
+
if (!normalizedQuery || terms.length === 0) return [];
|
|
68
|
+
const inputs = [];
|
|
69
|
+
for (const entry of entries) {
|
|
70
|
+
throwIfAborted(signal);
|
|
71
|
+
for (const passage of splitTextIntoPassages(entry.text, signal)) {
|
|
72
|
+
const normalizedPassage = normalizeText(passage.text);
|
|
73
|
+
inputs.push({
|
|
74
|
+
entry,
|
|
75
|
+
passage,
|
|
76
|
+
exactPhrase: normalizedPassage.includes(normalizedQuery),
|
|
77
|
+
matchedTerms: terms.filter((term) => passage.tokens.includes(term)),
|
|
78
|
+
});
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
return rankInputs(inputs, terms, signal);
|
|
82
|
+
}
|
|
83
|
+
export function strongestTextPassage(text, query) {
|
|
84
|
+
const normalizedQuery = normalizeText(query);
|
|
85
|
+
const terms = unique(tokenize(query));
|
|
86
|
+
if (!normalizedQuery || terms.length === 0) return undefined;
|
|
87
|
+
const inputs = splitTextIntoPassages(text).map((passage) => ({
|
|
88
|
+
passage,
|
|
89
|
+
exactPhrase: normalizeText(passage.text).includes(normalizedQuery),
|
|
90
|
+
matchedTerms: terms.filter((term) => passage.tokens.includes(term)),
|
|
91
|
+
}));
|
|
92
|
+
return rankInputs(inputs, terms)[0];
|
|
93
|
+
}
|
|
94
|
+
export function focusedWindow(text, focus, maxCharacters) {
|
|
95
|
+
const strongest = strongestTextPassage(text, focus);
|
|
96
|
+
if (!strongest) return undefined;
|
|
97
|
+
const center = Math.floor((strongest.passage.start + strongest.passage.end) / 2);
|
|
98
|
+
const proposedStart = Math.max(0, Math.min(center - Math.floor(maxCharacters / 2), text.length - maxCharacters));
|
|
99
|
+
const previousLineBreak = proposedStart > 0 ? text.lastIndexOf("\n", proposedStart - 1) : -1;
|
|
100
|
+
const start = previousLineBreak >= 0 ? previousLineBreak + 1 : proposedStart;
|
|
101
|
+
const proposedEnd = Math.min(text.length, start + maxCharacters);
|
|
102
|
+
const previousEndLineBreak = proposedEnd < text.length ? text.lastIndexOf("\n", proposedEnd) : -1;
|
|
103
|
+
const end = previousEndLineBreak > start ? previousEndLineBreak + 1 : proposedEnd;
|
|
104
|
+
return text.slice(start, end);
|
|
105
|
+
}
|