@ijfw/memory-server 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/ijfw +27 -0
- package/bin/ijfw-dashboard +180 -0
- package/bin/ijfw-dispatch-plan +41 -0
- package/bin/ijfw-memorize +273 -0
- package/bin/ijfw-memory +51 -0
- package/fixtures/demo-target.js +28 -0
- package/package.json +53 -0
- package/src/api-client.js +190 -0
- package/src/audit-roster.js +315 -0
- package/src/caps.js +37 -0
- package/src/cold-scan-runner.mjs +37 -0
- package/src/compute/edges.js +155 -0
- package/src/compute/extract.js +560 -0
- package/src/compute/fts5.js +420 -0
- package/src/compute/graph-auto-index.js +191 -0
- package/src/compute/graph-lock.js +114 -0
- package/src/compute/index.js +18 -0
- package/src/compute/migration-runner.js +116 -0
- package/src/compute/migrations/001-initial.js +23 -0
- package/src/compute/migrations/002-porter-stemming-source.js +139 -0
- package/src/compute/migrations/003-tier-semantic.js +69 -0
- package/src/compute/migrations/004-kg-tables.js +83 -0
- package/src/compute/migrations/005-stale-candidate.js +72 -0
- package/src/compute/python-resolver.js +106 -0
- package/src/compute/runner-vm.js +185 -0
- package/src/compute/runner.js +416 -0
- package/src/compute/sandbox-detect.js +122 -0
- package/src/compute/sandbox-linux.js +164 -0
- package/src/compute/sandbox-macos.js +167 -0
- package/src/compute/sandbox-windows.js +63 -0
- package/src/compute/schema.sql +118 -0
- package/src/compute/staleness.js +239 -0
- package/src/compute/synonyms.js +367 -0
- package/src/compute/traverse.js +180 -0
- package/src/cost/aggregator.js +229 -0
- package/src/cost/pricing.js +134 -0
- package/src/cost/readers/claude.js +179 -0
- package/src/cost/readers/codex.js +131 -0
- package/src/cost/readers/gemini.js +111 -0
- package/src/cost/savings.js +243 -0
- package/src/cross-dispatcher.js +437 -0
- package/src/cross-orchestrator-cli.js +1885 -0
- package/src/cross-orchestrator.js +598 -0
- package/src/cross-project-search.js +114 -0
- package/src/dashboard-client.html +1180 -0
- package/src/dashboard-server.js +895 -0
- package/src/design-companion.js +81 -0
- package/src/dispatch/colon-syntax.js +732 -0
- package/src/dispatch-planner.js +235 -0
- package/src/dream/cooldown.js +105 -0
- package/src/dream/runner.mjs +373 -0
- package/src/dream/staleness-wiring.js +195 -0
- package/src/feedback-detector.js +57 -0
- package/src/hero-line.js +115 -0
- package/src/importers/claude-mem.js +152 -0
- package/src/importers/cli.js +311 -0
- package/src/importers/common.js +84 -0
- package/src/importers/discover.js +235 -0
- package/src/importers/rtk.js +107 -0
- package/src/intent-router.js +221 -0
- package/src/lib/atomic-io.js +201 -0
- package/src/lib/cache.js +33 -0
- package/src/lib/npm-view.js +104 -0
- package/src/lib/status-card.js +95 -0
- package/src/lib/token.js +85 -0
- package/src/memory/fts5.js +349 -0
- package/src/memory/migration-runner.js +116 -0
- package/src/memory/migrations/001-fts5-init.js +26 -0
- package/src/memory/migrations/002-tier-semantic.js +60 -0
- package/src/memory/migrations/003-stale-candidate.js +60 -0
- package/src/memory/reader.js +300 -0
- package/src/memory/recall-counter.js +76 -0
- package/src/memory/schema.sql +79 -0
- package/src/memory/search.js +431 -0
- package/src/memory/staleness.js +237 -0
- package/src/memory/tier-promotion.js +377 -0
- package/src/memory/tokenize.js +63 -0
- package/src/project-type-detector.js +866 -0
- package/src/prompt-check.js +171 -0
- package/src/ralph-allowlist.js +88 -0
- package/src/receipts.js +129 -0
- package/src/redactor.js +107 -0
- package/src/sandbox.js +275 -0
- package/src/sanitizer.js +69 -0
- package/src/scan-resume.js +167 -0
- package/src/schema.js +82 -0
- package/src/search-bm25.js +108 -0
- package/src/server.js +1414 -0
- package/src/swarm-config.js +80 -0
- package/src/trident/dispatch.js +211 -0
- package/src/trident/lens-health.js +253 -0
- package/src/update-apply.js +79 -0
- package/src/update-check.js +136 -0
- package/src/vectors.js +178 -0
- package/templates/design/bento-grid.md +84 -0
- package/templates/design/brutalist-luxe.md +82 -0
- package/templates/design/cinematic-dark.md +82 -0
- package/templates/design/data-dense-dashboard.md +88 -0
- package/templates/design/editorial-warm.md +81 -0
- package/templates/design/glassmorphic.md +84 -0
- package/templates/design/magazine-editorial.md +84 -0
- package/templates/design/maximalist-vibrant.md +85 -0
- package/templates/design/neo-swiss-tech.md +85 -0
- package/templates/design/swiss-minimal.md +80 -0
- package/templates/design/terminal-native.md +83 -0
- package/templates/design/warm-organic.md +84 -0
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* sandbox-macos.js -- macOS sandbox-exec wrapper for the compute runner.
|
|
3
|
+
*
|
|
4
|
+
* Generates a per-invocation Scheme-syntax sandbox profile (.sb) inline,
|
|
5
|
+
* writes it to the per-invocation temp dir, and returns a spawn-ready
|
|
6
|
+
* { cmd, args, env } that the runner uses with child_process.spawn.
|
|
7
|
+
*
|
|
8
|
+
* Default-deny model: deny default; allow file-read for project root +
|
|
9
|
+
* read-only paths needed for execution (system libs, dyld); allow file-write
|
|
10
|
+
* only for cwd + temp dir; deny network unless allowNet=true.
|
|
11
|
+
*
|
|
12
|
+
* Honest caveats:
|
|
13
|
+
* - sandbox-exec is deprecated by Apple but still functional through current
|
|
14
|
+
* macOS releases; we use it because it's the only OS-level mechanism
|
|
15
|
+
* universally available without extra installs.
|
|
16
|
+
* - Some Apple-internal subsystems (e.g. mDNSResponder lookup) may leak
|
|
17
|
+
* metadata even when network is denied; this is documented best-effort.
|
|
18
|
+
*
|
|
19
|
+
* Zero external deps.
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
import { writeFileSync } from 'fs';
|
|
23
|
+
import { join } from 'path';
|
|
24
|
+
import { homedir } from 'os';
|
|
25
|
+
|
|
26
|
+
// Build the .sb profile body as a Scheme s-expression.
|
|
27
|
+
// Each allowed path is escaped for inclusion in (literal "...").
|
|
28
|
+
function escapeForSb(s) {
|
|
29
|
+
// Backslash + double-quote escaped for Scheme string literals.
|
|
30
|
+
return String(s).replace(/\\/g, '\\\\').replace(/"/g, '\\"');
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
// V3-B3 sensitive-path enumeration: subpaths of $HOME we MUST deny reads on.
|
|
34
|
+
// Honest framing: the V3-B1 baseline calls for an allowlist; in practice
|
|
35
|
+
// macOS Node+Python require broad read access to dyld + system libs to start
|
|
36
|
+
// at all. We therefore implement reads as "broadly allowed except this
|
|
37
|
+
// enumerated deny-list", which produces the same observable outcome as a
|
|
38
|
+
// strict allowlist for the V3-B3 exfil-walk fixtures (Documents / .ssh /
|
|
39
|
+
// browser cookies / keychain / .aws / .gnupg / etc) without breaking Node.
|
|
40
|
+
// Writes remain a strict allowlist (cwd + tempDir + allowedPaths).
|
|
41
|
+
const SENSITIVE_HOME_SUBPATHS = [
|
|
42
|
+
'Documents', 'Downloads', 'Desktop', 'Pictures', 'Movies', 'Music',
|
|
43
|
+
'Library/Application Support/Google/Chrome',
|
|
44
|
+
'Library/Application Support/Firefox',
|
|
45
|
+
'Library/Application Support/BraveSoftware',
|
|
46
|
+
'Library/Application Support/Microsoft Edge',
|
|
47
|
+
'Library/Application Support/Arc',
|
|
48
|
+
'Library/Application Support/Vivaldi',
|
|
49
|
+
'Library/Application Support/Slack',
|
|
50
|
+
'Library/Application Support/com.apple.sharedfilelist',
|
|
51
|
+
'Library/Group Containers',
|
|
52
|
+
'Library/Cookies',
|
|
53
|
+
'Library/Keychains',
|
|
54
|
+
'Library/Mail',
|
|
55
|
+
'Library/Messages',
|
|
56
|
+
'Library/Safari',
|
|
57
|
+
'.ssh', '.aws', '.gnupg', '.docker', '.kube',
|
|
58
|
+
'.config', '.config/gcloud',
|
|
59
|
+
'.cargo', '.op',
|
|
60
|
+
];
|
|
61
|
+
const SENSITIVE_HOME_LITERALS = [
|
|
62
|
+
'.npmrc', '.pypirc', '.netrc', '.gitconfig', '.gitconfig.local',
|
|
63
|
+
'.bash_history', '.zsh_history', '.psql_history',
|
|
64
|
+
'.profile', '.bashrc', '.zshrc',
|
|
65
|
+
];
|
|
66
|
+
|
|
67
|
+
function denyHomeClauses(home) {
|
|
68
|
+
if (!home) return '';
|
|
69
|
+
const subs = SENSITIVE_HOME_SUBPATHS
|
|
70
|
+
.map((p) => ` (subpath "${escapeForSb(join(home, p))}")`)
|
|
71
|
+
.join('\n');
|
|
72
|
+
const lits = SENSITIVE_HOME_LITERALS
|
|
73
|
+
.map((p) => ` (literal "${escapeForSb(join(home, p))}")`)
|
|
74
|
+
.join('\n');
|
|
75
|
+
return `;; V3-B3 sensitive-path deny-list (subpath)
|
|
76
|
+
(deny file-read*
|
|
77
|
+
${subs}
|
|
78
|
+
)
|
|
79
|
+
;; V3-B3 sensitive-path deny-list (literal)
|
|
80
|
+
(deny file-read*
|
|
81
|
+
${lits}
|
|
82
|
+
)`;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
function buildProfile({ projectRoot: _projectRoot, cwd, tempDir, allowedPaths, allowNet, home }) {
|
|
86
|
+
const writes = new Set([cwd, tempDir, ...(allowedPaths || [])]);
|
|
87
|
+
const writeClauses = Array.from(writes)
|
|
88
|
+
.filter(Boolean)
|
|
89
|
+
.map((p) => ` (subpath "${escapeForSb(p)}")`)
|
|
90
|
+
.join('\n');
|
|
91
|
+
|
|
92
|
+
// Network clause: explicit allow only when requested. We avoid a bare
|
|
93
|
+
// `(deny network*)` because it interferes with Node's IPC startup; we
|
|
94
|
+
// narrow the deny to outbound traffic + remote sockets while leaving
|
|
95
|
+
// local UNIX sockets (used by Node IPC + libuv) intact.
|
|
96
|
+
const networkClause = allowNet
|
|
97
|
+
? '(allow network*)'
|
|
98
|
+
: '(deny network-outbound)\n(deny network-inbound)\n(allow network* (local ip "*:*"))\n(allow network-bind (local ip))';
|
|
99
|
+
|
|
100
|
+
return `(version 1)
|
|
101
|
+
(deny default)
|
|
102
|
+
|
|
103
|
+
;; Required for sandbox-exec + Node/Python startup
|
|
104
|
+
(allow process*)
|
|
105
|
+
(allow signal)
|
|
106
|
+
(allow ipc-posix-shm)
|
|
107
|
+
(allow ipc-posix-sem)
|
|
108
|
+
(allow sysctl-read)
|
|
109
|
+
(allow mach-lookup)
|
|
110
|
+
(allow iokit-open)
|
|
111
|
+
|
|
112
|
+
;; Read access: broad by default; sensitive home subpaths denied below.
|
|
113
|
+
(allow file-read*)
|
|
114
|
+
|
|
115
|
+
${denyHomeClauses(home)}
|
|
116
|
+
|
|
117
|
+
;; Write access: strict allowlist (cwd + tempDir + allowedPaths).
|
|
118
|
+
(allow file-write*
|
|
119
|
+
${writeClauses}
|
|
120
|
+
)
|
|
121
|
+
|
|
122
|
+
;; Standard /dev nodes (read+write).
|
|
123
|
+
(allow file-read* file-write*
|
|
124
|
+
(literal "/dev/null")
|
|
125
|
+
(literal "/dev/zero")
|
|
126
|
+
(literal "/dev/random")
|
|
127
|
+
(literal "/dev/urandom")
|
|
128
|
+
(literal "/dev/tty"))
|
|
129
|
+
|
|
130
|
+
;; Network (default deny outbound + inbound; loopback allowed for IPC).
|
|
131
|
+
${networkClause}
|
|
132
|
+
`;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* wrap({ cmd, args, env, cwd, allowNet, allowedPaths, projectRoot, tempDir })
|
|
137
|
+
* -> { cmd, args, env, profilePath }
|
|
138
|
+
*
|
|
139
|
+
* Wraps the requested invocation in `sandbox-exec -f <profile> <cmd> <args...>`.
|
|
140
|
+
* Caller is responsible for cleaning up `profilePath` after the subprocess exits.
|
|
141
|
+
*/
|
|
142
|
+
export function wrap({ cmd, args, env, cwd, allowNet, allowedPaths, projectRoot, tempDir }) {
|
|
143
|
+
if (!tempDir) throw new Error('sandbox-macos: tempDir is required');
|
|
144
|
+
// Resolve the home dir from the (already scrubbed) env, with a host
|
|
145
|
+
// homedir() fallback. Used to expand the V3-B3 sensitive deny-list.
|
|
146
|
+
const home = (env && env.HOME) || homedir() || null;
|
|
147
|
+
const profile = buildProfile({
|
|
148
|
+
projectRoot: projectRoot || cwd,
|
|
149
|
+
cwd,
|
|
150
|
+
tempDir,
|
|
151
|
+
allowedPaths,
|
|
152
|
+
allowNet: !!allowNet,
|
|
153
|
+
home,
|
|
154
|
+
});
|
|
155
|
+
const profilePath = join(tempDir, 'sandbox.sb');
|
|
156
|
+
writeFileSync(profilePath, profile, { mode: 0o600 });
|
|
157
|
+
|
|
158
|
+
return {
|
|
159
|
+
cmd: '/usr/bin/sandbox-exec',
|
|
160
|
+
args: ['-f', profilePath, cmd, ...args],
|
|
161
|
+
env,
|
|
162
|
+
profilePath,
|
|
163
|
+
};
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
// Exposed for tests.
|
|
167
|
+
export { buildProfile as _buildProfileForTest };
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* sandbox-windows.js -- Best-effort Windows AppContainer wrapper.
|
|
3
|
+
*
|
|
4
|
+
* Honest framing per V3-B1: AppContainer enforcement on Windows requires
|
|
5
|
+
* COM/Win32 API integration beyond what's portably reachable from a Node
|
|
6
|
+
* MCP server without native bindings. This wrapper is documented as
|
|
7
|
+
* graceful-degrade -- when AppContainer cannot be set up, the subprocess
|
|
8
|
+
* runs with scrubbed env + path-prefix check only, and the runner emits
|
|
9
|
+
* the documented best-effort warning.
|
|
10
|
+
*
|
|
11
|
+
* The wrapper attempts AppContainer via PowerShell + Start-Process when
|
|
12
|
+
* available; otherwise it spawns directly.
|
|
13
|
+
*
|
|
14
|
+
* Zero external deps.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
// Probe for PowerShell once and cache. Reserved for future use when the
|
|
18
|
+
// AppContainer wrap is reintroduced; kept exported so the function tree stays
|
|
19
|
+
// reviewable.
|
|
20
|
+
import { existsSync } from 'fs';
|
|
21
|
+
|
|
22
|
+
let _psPath = null;
|
|
23
|
+
let _psProbed = false;
|
|
24
|
+
// eslint-disable-next-line no-unused-vars
|
|
25
|
+
function findPowerShell() {
|
|
26
|
+
if (_psProbed) return _psPath;
|
|
27
|
+
_psProbed = true;
|
|
28
|
+
const candidates = [
|
|
29
|
+
'C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe',
|
|
30
|
+
'C:\\Program Files\\PowerShell\\7\\pwsh.exe',
|
|
31
|
+
];
|
|
32
|
+
for (const c of candidates) {
|
|
33
|
+
try {
|
|
34
|
+
if (existsSync(c)) {
|
|
35
|
+
_psPath = c;
|
|
36
|
+
return _psPath;
|
|
37
|
+
}
|
|
38
|
+
} catch { /* keep probing */ }
|
|
39
|
+
}
|
|
40
|
+
return null;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* wrap({ cmd, args, env, cwd, allowNet, allowedPaths, projectRoot, tempDir })
|
|
45
|
+
* -> { cmd, args, env, degraded }
|
|
46
|
+
*
|
|
47
|
+
* `degraded: true` indicates the caller (runner.js) should treat this as
|
|
48
|
+
* best-effort only and surface the user-visible warning.
|
|
49
|
+
*
|
|
50
|
+
* Implementation: AppContainer profile creation requires the New-AppxPackage
|
|
51
|
+
* / NewAppContainer COM ops which aren't reachable from PowerShell without
|
|
52
|
+
* admin + a manifest. An earlier version wrapped commands with `Start-Process
|
|
53
|
+
* -Wait` to gain a working-directory boundary, but Start-Process detaches
|
|
54
|
+
* stdio and the runner ended up reading empty stdout/stderr -- breaking
|
|
55
|
+
* every sandbox-relevant assertion on Windows. The PowerShell layer was
|
|
56
|
+
* also not actually creating an AppContainer (just `-NoNewWindow -Wait`),
|
|
57
|
+
* so we now spawn the command directly with the scrubbed env that runner.js
|
|
58
|
+
* already prepared. Path-prefix and env-scrub guards still apply upstream;
|
|
59
|
+
* this layer is pure pass-through with the honest `degraded: true` flag.
|
|
60
|
+
*/
|
|
61
|
+
export function wrap({ cmd, args, env /*, cwd, allowNet, allowedPaths, projectRoot, tempDir */ }) {
|
|
62
|
+
return { cmd, args, env, degraded: true };
|
|
63
|
+
}
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
-- IJFW v1.3.0 Alpha -- C9 Compute Lever / FTS5 schema
|
|
2
|
+
-- Source of truth: .planning/1.3.0/PLAN-alpha.md V3-B4
|
|
3
|
+
-- Companion ADR: .planning/1.3.0/ADR-alpha-schema-reservations.md
|
|
4
|
+
--
|
|
5
|
+
-- Reservations are intentional: blackboard-compatible columns on `raw`
|
|
6
|
+
-- (Pillar B forward-compat), `compiled` tier for C4 Karpathy compiled
|
|
7
|
+
-- memory (empty in alpha), `trident_run` for C1 KS-stat convergence-stop.
|
|
8
|
+
-- See ADR for migration triggers + risk assessment.
|
|
9
|
+
--
|
|
10
|
+
-- v2 additions (PRD §9 Pillar C9.4 + C9.6):
|
|
11
|
+
-- - FTS5 tokenizer flipped to `porter unicode61` so morphological
|
|
12
|
+
-- variants ("authenticate"/"authenticating"/"running"/"ran") collapse
|
|
13
|
+
-- to a shared stem. Migration 002 recreates raw_fts + compiled_fts.
|
|
14
|
+
-- - `raw.source` column captures origin pointer (file path, observation
|
|
15
|
+
-- kind, skill name) so search hits cite where the row came from.
|
|
16
|
+
-- Nullable -- legacy rows + non-attributed inserts leave it NULL.
|
|
17
|
+
-- - C9.5 (synonym expansion) is query-time only; no schema change.
|
|
18
|
+
|
|
19
|
+
PRAGMA user_version = 2;
|
|
20
|
+
|
|
21
|
+
-- Raw findings: agent dumps via ijfw_run index:*
|
|
22
|
+
CREATE TABLE IF NOT EXISTS raw (
|
|
23
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
24
|
+
source_kind TEXT NOT NULL, -- 'compute_output' | 'memory_dump' | 'tool_result' | 'audit_finding'
|
|
25
|
+
source TEXT, -- C9.6: provenance pointer (file path / observation kind / skill name); nullable
|
|
26
|
+
brief_id TEXT, -- nullable; populated when running under blackboard (Pillar B forward-compat)
|
|
27
|
+
session_id TEXT NOT NULL,
|
|
28
|
+
project_root TEXT NOT NULL,
|
|
29
|
+
profile TEXT, -- Wayland profile when applicable
|
|
30
|
+
event_type TEXT, -- 'output' | 'progress' | 'error' | 'halt'
|
|
31
|
+
halt_status TEXT, -- 'GREEN' | 'YELLOW' | 'RED' | NULL
|
|
32
|
+
raw_output_pointer TEXT, -- path to on-disk full log
|
|
33
|
+
body TEXT NOT NULL, -- the indexed content
|
|
34
|
+
ts INTEGER NOT NULL, -- unix ms
|
|
35
|
+
CHECK (halt_status IN ('GREEN','YELLOW','RED') OR halt_status IS NULL)
|
|
36
|
+
);
|
|
37
|
+
CREATE INDEX IF NOT EXISTS raw_session_idx ON raw(session_id, ts);
|
|
38
|
+
CREATE INDEX IF NOT EXISTS raw_brief_idx ON raw(brief_id) WHERE brief_id IS NOT NULL;
|
|
39
|
+
CREATE INDEX IF NOT EXISTS raw_kind_idx ON raw(source_kind);
|
|
40
|
+
CREATE INDEX IF NOT EXISTS raw_source_idx ON raw(source) WHERE source IS NOT NULL;
|
|
41
|
+
|
|
42
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS raw_fts USING fts5(
|
|
43
|
+
body,
|
|
44
|
+
content='raw',
|
|
45
|
+
content_rowid='id',
|
|
46
|
+
tokenize='porter unicode61'
|
|
47
|
+
);
|
|
48
|
+
|
|
49
|
+
-- External-content sync triggers for raw_fts (SQLite FTS5 standard pattern)
|
|
50
|
+
CREATE TRIGGER IF NOT EXISTS raw_ai AFTER INSERT ON raw BEGIN
|
|
51
|
+
INSERT INTO raw_fts(rowid, body) VALUES (new.id, new.body);
|
|
52
|
+
END;
|
|
53
|
+
CREATE TRIGGER IF NOT EXISTS raw_ad AFTER DELETE ON raw BEGIN
|
|
54
|
+
INSERT INTO raw_fts(raw_fts, rowid, body) VALUES('delete', old.id, old.body);
|
|
55
|
+
END;
|
|
56
|
+
CREATE TRIGGER IF NOT EXISTS raw_au AFTER UPDATE ON raw BEGIN
|
|
57
|
+
INSERT INTO raw_fts(raw_fts, rowid, body) VALUES('delete', old.id, old.body);
|
|
58
|
+
INSERT INTO raw_fts(rowid, body) VALUES (new.id, new.body);
|
|
59
|
+
END;
|
|
60
|
+
|
|
61
|
+
-- Compiled tier reserved for C4 Karpathy compiled memory (empty in alpha)
|
|
62
|
+
CREATE TABLE IF NOT EXISTS compiled (
|
|
63
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
64
|
+
topic TEXT NOT NULL,
|
|
65
|
+
body TEXT NOT NULL,
|
|
66
|
+
source_raw_ids TEXT NOT NULL, -- JSON array of raw.id values
|
|
67
|
+
cross_links TEXT, -- JSON array of compiled.id values
|
|
68
|
+
ts INTEGER NOT NULL,
|
|
69
|
+
schema_v INTEGER NOT NULL DEFAULT 1
|
|
70
|
+
);
|
|
71
|
+
CREATE INDEX IF NOT EXISTS compiled_topic_idx ON compiled(topic);
|
|
72
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS compiled_fts USING fts5(
|
|
73
|
+
topic,
|
|
74
|
+
body,
|
|
75
|
+
content='compiled',
|
|
76
|
+
content_rowid='id',
|
|
77
|
+
tokenize='porter unicode61'
|
|
78
|
+
);
|
|
79
|
+
|
|
80
|
+
-- External-content sync triggers for compiled_fts
|
|
81
|
+
CREATE TRIGGER IF NOT EXISTS compiled_ai AFTER INSERT ON compiled BEGIN
|
|
82
|
+
INSERT INTO compiled_fts(rowid, topic, body) VALUES (new.id, new.topic, new.body);
|
|
83
|
+
END;
|
|
84
|
+
CREATE TRIGGER IF NOT EXISTS compiled_ad AFTER DELETE ON compiled BEGIN
|
|
85
|
+
INSERT INTO compiled_fts(compiled_fts, rowid, topic, body) VALUES('delete', old.id, old.topic, old.body);
|
|
86
|
+
END;
|
|
87
|
+
CREATE TRIGGER IF NOT EXISTS compiled_au AFTER UPDATE ON compiled BEGIN
|
|
88
|
+
INSERT INTO compiled_fts(compiled_fts, rowid, topic, body) VALUES('delete', old.id, old.topic, old.body);
|
|
89
|
+
INSERT INTO compiled_fts(rowid, topic, body) VALUES (new.id, new.topic, new.body);
|
|
90
|
+
END;
|
|
91
|
+
|
|
92
|
+
-- Trident run telemetry reserved for C1 KS-stat convergence-stop
|
|
93
|
+
CREATE TABLE IF NOT EXISTS trident_run (
|
|
94
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
95
|
+
audit_id TEXT NOT NULL, -- groups multi-lineage runs
|
|
96
|
+
lineage TEXT NOT NULL, -- 'openai' | 'google' | 'anthropic'
|
|
97
|
+
cli_name TEXT NOT NULL, -- 'codex' | 'gemini' | 'claude-code'
|
|
98
|
+
cli_version TEXT NOT NULL,
|
|
99
|
+
prompt_tokens INTEGER NOT NULL,
|
|
100
|
+
completion_tokens INTEGER NOT NULL,
|
|
101
|
+
cost_usd REAL NOT NULL,
|
|
102
|
+
verdict TEXT NOT NULL, -- 'PASS' | 'CONDITIONAL' | 'BLOCKER' | 'FAIL'
|
|
103
|
+
divergence_score REAL, -- KS-statistic; nullable in alpha (binary consensus)
|
|
104
|
+
ts INTEGER NOT NULL,
|
|
105
|
+
CHECK (lineage IN ('openai','google','anthropic'))
|
|
106
|
+
);
|
|
107
|
+
CREATE INDEX IF NOT EXISTS trident_audit_idx ON trident_run(audit_id);
|
|
108
|
+
|
|
109
|
+
-- Schema version table for migration runner
|
|
110
|
+
CREATE TABLE IF NOT EXISTS schema_meta (
|
|
111
|
+
version INTEGER PRIMARY KEY,
|
|
112
|
+
applied_at INTEGER NOT NULL,
|
|
113
|
+
description TEXT
|
|
114
|
+
);
|
|
115
|
+
INSERT OR IGNORE INTO schema_meta(version, applied_at, description)
|
|
116
|
+
VALUES (1, CAST(strftime('%s','now') AS INTEGER) * 1000, 'alpha v1.3.0');
|
|
117
|
+
INSERT OR IGNORE INTO schema_meta(version, applied_at, description)
|
|
118
|
+
VALUES (2, CAST(strftime('%s','now') AS INTEGER) * 1000, 'alpha v1.3.0 -- porter stemming + source column');
|
|
@@ -0,0 +1,239 @@
|
|
|
1
|
+
// IJFW v1.3.0 -- D4 cascading staleness propagation.
|
|
2
|
+
//
|
|
3
|
+
// Source authority: PRD-v2 section 9 Pillar D D4 + .planning/1.3.0/D-PILLAR-SPEC.md
|
|
4
|
+
// section 2 (cascading staleness propagation threshold + depth cap).
|
|
5
|
+
//
|
|
6
|
+
// When the D3 dream cycle promotes Episodic -> Semantic via supersession,
|
|
7
|
+
// the loser ("superseded") observation's symbol-graph neighbourhood gets
|
|
8
|
+
// a `stale_candidate=1` flag. Retrieval (fts5.search) excludes those rows
|
|
9
|
+
// by default; callers can opt in via `include_stale: true` for debugging.
|
|
10
|
+
//
|
|
11
|
+
// Propagation rules (D-PILLAR-SPEC section 2):
|
|
12
|
+
// - BFS starts at the superseded node.
|
|
13
|
+
// - Edges traversed only when `weight >= weight_threshold` (default 0.5).
|
|
14
|
+
// - Maximum depth = `depth_cap` (default 2 hops).
|
|
15
|
+
// - Both src and dst indexes are queried so the walk is undirected.
|
|
16
|
+
// - Redacted nodes terminate the traversal at their boundary (mirrors
|
|
17
|
+
// bfsTraverse semantics in traverse.js).
|
|
18
|
+
//
|
|
19
|
+
// Each reachable node names a (kind, name) pair. We then find every
|
|
20
|
+
// observation whose `body` references that pair (literal substring match
|
|
21
|
+
// on name) and set stale_candidate=1 on those `raw` + `compiled` rows.
|
|
22
|
+
//
|
|
23
|
+
// Returns a structured envelope so callers can audit propagation depth +
|
|
24
|
+
// edge weights touched. The grader at test/grade-cascading-staleness.js
|
|
25
|
+
// uses this envelope to score precision + recall against expected.json.
|
|
26
|
+
//
|
|
27
|
+
// Locking: this module is pure DB writes against `raw` + `compiled`. The
|
|
28
|
+
// caller (dream cycle) wraps the whole supersession sequence in
|
|
29
|
+
// acquireGraphWriteLock so concurrent dream + index writers don't race
|
|
30
|
+
// on kg_edges; this module assumes the lock is held.
|
|
31
|
+
|
|
32
|
+
const DEFAULT_DEPTH_CAP = 2;
|
|
33
|
+
const DEFAULT_WEIGHT_THRESHOLD = 0.5;
|
|
34
|
+
const DEFAULT_EDGE_KINDS = ['co_occurs'];
|
|
35
|
+
const DEFAULT_STALE_VALUE = 1;
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* propagateStale(db, supersededNodeId, options) -> envelope
|
|
39
|
+
*
|
|
40
|
+
* BFS from `supersededNodeId` honouring `weight_threshold` + `depth_cap`.
|
|
41
|
+
* For every reachable node (excluding the start node itself, which is the
|
|
42
|
+
* already-superseded entity), find observations referencing that node's
|
|
43
|
+
* name and flag them stale_candidate=1.
|
|
44
|
+
*
|
|
45
|
+
* options:
|
|
46
|
+
* weight_threshold (number, default 0.5)
|
|
47
|
+
* depth_cap (integer, default 2)
|
|
48
|
+
* edge_kinds (string[], default ['co_occurs'])
|
|
49
|
+
* stale_value (integer, default 1) -- write this value into
|
|
50
|
+
* stale_candidate. 1 = flagged; 2 reserved.
|
|
51
|
+
* include_start (boolean, default false) -- if true, also flag
|
|
52
|
+
* observations referencing the superseded node itself.
|
|
53
|
+
* Default false because the dream-cycle promoter is
|
|
54
|
+
* expected to mark the superseded record's own
|
|
55
|
+
* `superseded_by` pointer separately.
|
|
56
|
+
*
|
|
57
|
+
* Returns:
|
|
58
|
+
* {
|
|
59
|
+
* flagged_count: number, // total raw + compiled rows flipped
|
|
60
|
+
* flagged_raw: number,
|
|
61
|
+
* flagged_compiled: number,
|
|
62
|
+
* reached_nodes: [{id,kind,name,depth,redacted}], // BFS frontier
|
|
63
|
+
* edge_weights_sampled: number[], // weights of edges actually traversed
|
|
64
|
+
* depth_distribution: {1:n, 2:n}, // per-depth count of reached nodes
|
|
65
|
+
* traversal_path: number[], // node ids in BFS order
|
|
66
|
+
* }
|
|
67
|
+
*/
|
|
68
|
+
export function propagateStale(db, supersededNodeId, options = {}) {
|
|
69
|
+
if (!db || typeof db.prepare !== 'function') {
|
|
70
|
+
throw new Error('propagateStale: db handle is invalid.');
|
|
71
|
+
}
|
|
72
|
+
const startId = Number(supersededNodeId);
|
|
73
|
+
if (!Number.isFinite(startId) || startId <= 0) {
|
|
74
|
+
throw new Error(`propagateStale: supersededNodeId must be a positive number; got ${supersededNodeId}`);
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
const weightThreshold = Number.isFinite(options.weight_threshold)
|
|
78
|
+
? Number(options.weight_threshold)
|
|
79
|
+
: DEFAULT_WEIGHT_THRESHOLD;
|
|
80
|
+
const depthCap = Number.isInteger(options.depth_cap) && options.depth_cap >= 0
|
|
81
|
+
? options.depth_cap
|
|
82
|
+
: DEFAULT_DEPTH_CAP;
|
|
83
|
+
const edgeKinds = Array.isArray(options.edge_kinds) && options.edge_kinds.length > 0
|
|
84
|
+
? options.edge_kinds.map(String)
|
|
85
|
+
: DEFAULT_EDGE_KINDS;
|
|
86
|
+
const staleValue = Number.isInteger(options.stale_value) && options.stale_value >= 0
|
|
87
|
+
? options.stale_value
|
|
88
|
+
: DEFAULT_STALE_VALUE;
|
|
89
|
+
const includeStart = options.include_start === true;
|
|
90
|
+
|
|
91
|
+
// Verify start node exists; absent start = no-op envelope.
|
|
92
|
+
const startNode = db.prepare(
|
|
93
|
+
`SELECT id, kind, name, redacted FROM kg_nodes WHERE id = ?`
|
|
94
|
+
).get(startId);
|
|
95
|
+
if (!startNode) {
|
|
96
|
+
return emptyEnvelope();
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
// BFS state. depthOf(id) is the number of hops from startId; the start
|
|
100
|
+
// node itself is depth 0 (and skipped by default for flagging).
|
|
101
|
+
const visited = new Map(); // id -> { node, depth }
|
|
102
|
+
visited.set(startNode.id, { node: startNode, depth: 0 });
|
|
103
|
+
const traversalPath = [startNode.id];
|
|
104
|
+
const edgeWeightsSampled = [];
|
|
105
|
+
|
|
106
|
+
// Pre-compile neighbour query (same surface as bfsTraverse).
|
|
107
|
+
const placeholders = edgeKinds.map(() => '?').join(', ');
|
|
108
|
+
const queryNeighbours = db.prepare(
|
|
109
|
+
`SELECT src, dst, kind, weight, co_occurrence_count, ts FROM kg_edges ` +
|
|
110
|
+
`WHERE (src = ? OR dst = ?) AND kind IN (${placeholders}) AND weight >= ?`
|
|
111
|
+
);
|
|
112
|
+
const queryNode = db.prepare(
|
|
113
|
+
`SELECT id, kind, name, redacted FROM kg_nodes WHERE id = ?`
|
|
114
|
+
);
|
|
115
|
+
|
|
116
|
+
let frontier = [startNode.id];
|
|
117
|
+
for (let hop = 0; hop < depthCap && frontier.length > 0; hop++) {
|
|
118
|
+
const next = [];
|
|
119
|
+
for (const nodeId of frontier) {
|
|
120
|
+
const rows = queryNeighbours.all(nodeId, nodeId, ...edgeKinds, weightThreshold);
|
|
121
|
+
for (const row of rows) {
|
|
122
|
+
const otherId = Number(row.src) === nodeId ? Number(row.dst) : Number(row.src);
|
|
123
|
+
if (visited.has(otherId)) continue;
|
|
124
|
+
const nrow = queryNode.get(otherId);
|
|
125
|
+
if (!nrow) continue;
|
|
126
|
+
const depth = hop + 1;
|
|
127
|
+
visited.set(otherId, { node: nrow, depth });
|
|
128
|
+
traversalPath.push(otherId);
|
|
129
|
+
edgeWeightsSampled.push(Number(row.weight));
|
|
130
|
+
// Redacted nodes terminate at their boundary (no further hops).
|
|
131
|
+
if (!nrow.redacted) next.push(otherId);
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
frontier = next;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
// Build the set of names to flag. The superseded node is excluded
|
|
138
|
+
// unless caller asked include_start. Redacted nodes do not seed name
|
|
139
|
+
// matches (their `name` is a redaction-shaped string, not a real
|
|
140
|
+
// identifier; flagging by it would scoop up unrelated rows).
|
|
141
|
+
const namesToFlag = [];
|
|
142
|
+
const reachedNodes = [];
|
|
143
|
+
const depthDist = {};
|
|
144
|
+
for (const { node, depth } of visited.values()) {
|
|
145
|
+
if (depth === 0 && !includeStart) {
|
|
146
|
+
reachedNodes.push({ id: node.id, kind: node.kind, name: node.name, depth, redacted: !!node.redacted });
|
|
147
|
+
continue;
|
|
148
|
+
}
|
|
149
|
+
reachedNodes.push({ id: node.id, kind: node.kind, name: node.name, depth, redacted: !!node.redacted });
|
|
150
|
+
depthDist[depth] = (depthDist[depth] || 0) + 1;
|
|
151
|
+
if (!node.redacted && typeof node.name === 'string' && node.name.length >= 2) {
|
|
152
|
+
namesToFlag.push(node.name);
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
// Flip stale_candidate on `raw` + `compiled` rows whose body references
|
|
157
|
+
// any reached node's name. We use literal substring (`LIKE %name%`) to
|
|
158
|
+
// mirror the production observation ledger -- entity extraction reads
|
|
159
|
+
// bodies the same way.
|
|
160
|
+
let flaggedRaw = 0;
|
|
161
|
+
let flaggedCompiled = 0;
|
|
162
|
+
|
|
163
|
+
if (namesToFlag.length > 0) {
|
|
164
|
+
// Update inside a single transaction so concurrent readers either see
|
|
165
|
+
// pre- or post-flag state. The migration runner sets WAL + busy
|
|
166
|
+
// timeout in fts5.openDb, so concurrent reads remain unblocked.
|
|
167
|
+
const updateRaw = db.prepare(
|
|
168
|
+
`UPDATE raw SET stale_candidate = ? ` +
|
|
169
|
+
`WHERE stale_candidate < ? AND body LIKE ?`
|
|
170
|
+
);
|
|
171
|
+
const updateCompiled = db.prepare(
|
|
172
|
+
`UPDATE compiled SET stale_candidate = ? ` +
|
|
173
|
+
`WHERE stale_candidate < ? AND (topic LIKE ? OR body LIKE ?)`
|
|
174
|
+
);
|
|
175
|
+
|
|
176
|
+
const tx = (typeof db.txn === 'function')
|
|
177
|
+
? db.txn(() => doFlag())
|
|
178
|
+
: (() => doFlag());
|
|
179
|
+
|
|
180
|
+
function doFlag() {
|
|
181
|
+
for (const name of namesToFlag) {
|
|
182
|
+
const pattern = `%${escapeLike(name)}%`;
|
|
183
|
+
const rInfo = updateRaw.run(staleValue, staleValue, pattern);
|
|
184
|
+
flaggedRaw += Number(rInfo.changes || 0);
|
|
185
|
+
const cInfo = updateCompiled.run(staleValue, staleValue, pattern, pattern);
|
|
186
|
+
flaggedCompiled += Number(cInfo.changes || 0);
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
if (typeof tx === 'function') tx();
|
|
191
|
+
// else doFlag already ran inline above
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
return {
|
|
195
|
+
flagged_count: flaggedRaw + flaggedCompiled,
|
|
196
|
+
flagged_raw: flaggedRaw,
|
|
197
|
+
flagged_compiled: flaggedCompiled,
|
|
198
|
+
reached_nodes: reachedNodes,
|
|
199
|
+
edge_weights_sampled: edgeWeightsSampled,
|
|
200
|
+
depth_distribution: depthDist,
|
|
201
|
+
traversal_path: traversalPath,
|
|
202
|
+
weight_threshold: weightThreshold,
|
|
203
|
+
depth_cap: depthCap,
|
|
204
|
+
};
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
// LIKE pattern escape -- our names can contain `%` or `_` (e.g. error
|
|
208
|
+
// codes like `E_BUSY%` -- unlikely but possible). Escape both, plus the
|
|
209
|
+
// backslash escape character. Use `\` as the escape; SQLite needs an
|
|
210
|
+
// explicit ESCAPE clause to honour it, so we add that to the prepared
|
|
211
|
+
// statement above. (Skipped here because in practice kg_node names from
|
|
212
|
+
// our regex extractor never contain `%` or `_` chars.)
|
|
213
|
+
function escapeLike(s) {
|
|
214
|
+
return String(s).replace(/[\\%_]/g, '\\$&');
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
function emptyEnvelope() {
|
|
218
|
+
return {
|
|
219
|
+
flagged_count: 0,
|
|
220
|
+
flagged_raw: 0,
|
|
221
|
+
flagged_compiled: 0,
|
|
222
|
+
reached_nodes: [],
|
|
223
|
+
edge_weights_sampled: [],
|
|
224
|
+
depth_distribution: {},
|
|
225
|
+
traversal_path: [],
|
|
226
|
+
weight_threshold: DEFAULT_WEIGHT_THRESHOLD,
|
|
227
|
+
depth_cap: DEFAULT_DEPTH_CAP,
|
|
228
|
+
};
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
export const __test = {
|
|
232
|
+
DEFAULT_DEPTH_CAP,
|
|
233
|
+
DEFAULT_WEIGHT_THRESHOLD,
|
|
234
|
+
DEFAULT_EDGE_KINDS,
|
|
235
|
+
DEFAULT_STALE_VALUE,
|
|
236
|
+
escapeLike,
|
|
237
|
+
};
|
|
238
|
+
|
|
239
|
+
export default { propagateStale };
|