harness-everything 0.30.8 → 0.30.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/.github/workflows/ci.yml +31 -0
- package/CHANGELOG.md +21 -0
- package/README.md +1 -1
- package/benchmarks/results/live-host/codex-2026-10-13/README.md +55 -0
- package/benchmarks/results/live-host/codex-2026-10-13/observation-records.json +88 -0
- package/ci/doc-capability-consistency.test.js +11 -1
- package/ci/fixtures/system-one-resident-stub.py +7 -1
- package/ci/mechanism-2al-codex-hooks-compat.test.js +19 -0
- package/ci/mechanism-2am-observations-session-import.test.js +221 -0
- package/ci/mechanism-2ao-plugin-sync.test.js +129 -4
- package/ci/mechanism-2w-platform-compatibility.test.js +12 -0
- package/ci/mechanism-34-observations.test.js +64 -0
- package/ci/mechanism-system-one-resident.test.js +70 -3
- package/docs/platform-capabilities.md +8 -8
- package/docs/platform-compatibility.json +5 -5
- package/docs/system-one-observations.md +35 -8
- package/hooks/scripts/lib/observations.js +88 -12
- package/opencode-plugin/plugin.json +1 -1
- package/package.json +1 -1
- package/plugins/harness-everything/.codex-plugin/plugin.json +1 -1
- package/plugins/harness-everything/hooks/scripts/lib/observations.js +88 -12
- package/plugins/harness-everything/plugin.json +1 -1
- package/sbom.cdx.json +7 -7
- package/scripts/codex-hooks-compat.js +39 -14
- package/scripts/plugin-sync.js +100 -29
- package/scripts/system-one-observations-export.js +29 -2
- package/scripts/system-one-observations-import-codex-sessions.js +307 -0
- package/scripts/system-one-observations-review.js +11 -1
package/scripts/plugin-sync.js
CHANGED
|
@@ -177,20 +177,36 @@ function findMarketplace(payload, marketplace) {
|
|
|
177
177
|
}
|
|
178
178
|
|
|
179
179
|
function findPlugin(payload, plugin, marketplace) {
|
|
180
|
+
if (findPluginEntry(payload, plugin, marketplace)) return true;
|
|
181
|
+
if (hasExplicitUninstalledPlugin(payload, plugin, marketplace)) return false;
|
|
182
|
+
return collectStrings(payload).some(value => value.trim() === `${plugin}@${marketplace}`);
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
function pluginEntryMatches(value, keyHint, plugin, marketplace) {
|
|
180
186
|
const pluginId = `${plugin}@${marketplace}`;
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
187
|
+
const identifiers = [value.id, value.pluginId, value.identifier, keyHint];
|
|
188
|
+
if (identifiers.some(candidate => exactString(candidate, pluginId))) return true;
|
|
189
|
+
const names = [value.name, value.plugin, value.pluginName];
|
|
190
|
+
if (!names.some(candidate => exactString(candidate, plugin))) return false;
|
|
191
|
+
const sources = [
|
|
192
|
+
value.marketplace,
|
|
193
|
+
value.marketplaceName,
|
|
194
|
+
value.source && value.source.name,
|
|
195
|
+
value.source && value.source.id
|
|
196
|
+
].filter(Boolean);
|
|
197
|
+
return sources.length === 0 || sources.some(source => exactString(source, marketplace));
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
function findPluginEntry(payload, plugin, marketplace) {
|
|
201
|
+
return collectObjects(payload).find(({ value, keyHint }) =>
|
|
202
|
+
value.installed !== false && pluginEntryMatches(value, keyHint, plugin, marketplace)
|
|
203
|
+
)?.value || null;
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
function hasExplicitUninstalledPlugin(payload, plugin, marketplace) {
|
|
207
|
+
return collectObjects(payload).some(({ value, keyHint }) =>
|
|
208
|
+
value.installed === false && pluginEntryMatches(value, keyHint, plugin, marketplace)
|
|
209
|
+
);
|
|
194
210
|
}
|
|
195
211
|
|
|
196
212
|
function escapeRegExp(value) {
|
|
@@ -232,6 +248,11 @@ function createContext(options = {}) {
|
|
|
232
248
|
return result && result.status === 0 && !result.error;
|
|
233
249
|
})
|
|
234
250
|
};
|
|
251
|
+
context.codexHooksInstall = options.codexHooksInstall || (installOptions => require('./codex-hooks-compat').install({
|
|
252
|
+
...installOptions,
|
|
253
|
+
codexHome: options.codexHome,
|
|
254
|
+
env: options.env
|
|
255
|
+
}));
|
|
235
256
|
context.repositorySource = options.repositorySource || (
|
|
236
257
|
/^https?:\/\//.test(context.repository) || /^git@/.test(context.repository)
|
|
237
258
|
? context.repository
|
|
@@ -375,25 +396,30 @@ function queryCodexMarketplace(context) {
|
|
|
375
396
|
}
|
|
376
397
|
|
|
377
398
|
function queryCodexPlugin(context) {
|
|
378
|
-
const
|
|
399
|
+
const query = (jsonArgs, plainArgs) => listState(context, {
|
|
379
400
|
command: 'codex',
|
|
380
|
-
jsonArgs
|
|
381
|
-
plainArgs
|
|
382
|
-
detectJson: payload =>
|
|
401
|
+
jsonArgs,
|
|
402
|
+
plainArgs,
|
|
403
|
+
detectJson: payload => findPluginEntry(payload, context.plugin, context.marketplace) ||
|
|
404
|
+
(findPlugin(payload, context.plugin, context.marketplace) ? { detected: true } : null),
|
|
383
405
|
detectText: output => textHasPlugin(output, context.plugin, context.marketplace)
|
|
384
406
|
});
|
|
385
|
-
|
|
407
|
+
let state = query(
|
|
408
|
+
['plugin', 'list', '--marketplace', context.marketplace, '--json'],
|
|
409
|
+
['plugin', 'list', '--marketplace', context.marketplace]
|
|
410
|
+
);
|
|
386
411
|
|
|
387
412
|
// Older Codex builds may not accept --marketplace on plugin list. A full
|
|
388
413
|
// list still lets us fail closed while supporting those builds when JSON is
|
|
389
414
|
// available.
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
}
|
|
415
|
+
if (!state.known) {
|
|
416
|
+
state = query(['plugin', 'list', '--json'], ['plugin', 'list']);
|
|
417
|
+
}
|
|
418
|
+
if (state.known) {
|
|
419
|
+
state.plugin = state.value && typeof state.value === 'object' ? state.value : null;
|
|
420
|
+
state.value = Boolean(state.plugin || state.value);
|
|
421
|
+
}
|
|
422
|
+
return state;
|
|
397
423
|
}
|
|
398
424
|
|
|
399
425
|
function ensureClaudeMarketplace(context) {
|
|
@@ -452,7 +478,7 @@ function verifyPlugin(context, host, queryPlugin) {
|
|
|
452
478
|
const state = queryPlugin(context);
|
|
453
479
|
if (!state.known) return { ok: false, error: `${host} plugin state could not be verified: ${state.error}` };
|
|
454
480
|
if (!state.value) return { ok: false, error: `${host} reports ${context.plugin}@${context.marketplace} is not installed after synchronization` };
|
|
455
|
-
return { ok: true };
|
|
481
|
+
return { ok: true, state };
|
|
456
482
|
}
|
|
457
483
|
|
|
458
484
|
function syncClaude(context) {
|
|
@@ -477,6 +503,37 @@ function syncClaude(context) {
|
|
|
477
503
|
return { host: 'claude', status: 'synchronized', action };
|
|
478
504
|
}
|
|
479
505
|
|
|
506
|
+
function refreshCodexCompatibilityHooks(context, sourcePluginRoot) {
|
|
507
|
+
if (context.dryRun) return { ok: true, result: { status: 'dry-run', entries: 0, trustChanged: false } };
|
|
508
|
+
if (!sourcePluginRoot) {
|
|
509
|
+
return { ok: false, error: 'Codex did not report the installed plugin source path; refusing to copy hooks from the sync package' };
|
|
510
|
+
}
|
|
511
|
+
try {
|
|
512
|
+
const result = context.codexHooksInstall({ sourcePluginRoot });
|
|
513
|
+
if (!result || !['installed', 'updated'].includes(result.status)) {
|
|
514
|
+
return { ok: false, error: 'Codex compatibility-hook installer returned an unexpected result' };
|
|
515
|
+
}
|
|
516
|
+
return { ok: true, result };
|
|
517
|
+
} catch (error) {
|
|
518
|
+
return { ok: false, error: error && error.message ? error.message : String(error) };
|
|
519
|
+
}
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
function codexLocalPluginRoot(plugin) {
|
|
523
|
+
const source = plugin && plugin.source;
|
|
524
|
+
if (!source || typeof source !== 'object') {
|
|
525
|
+
return { ok: false, error: 'Codex did not report a tagged installed plugin source' };
|
|
526
|
+
}
|
|
527
|
+
if (source.source !== 'local') {
|
|
528
|
+
const variant = typeof source.source === 'string' ? source.source : 'missing';
|
|
529
|
+
return { ok: false, error: `Codex installed plugin source variant "${variant}" is unsupported; compatibility hooks require source "local"` };
|
|
530
|
+
}
|
|
531
|
+
if (typeof source.path !== 'string' || !source.path.trim() || !path.isAbsolute(source.path)) {
|
|
532
|
+
return { ok: false, error: 'Codex local plugin source must report an absolute filesystem path' };
|
|
533
|
+
}
|
|
534
|
+
return { ok: true, path: source.path };
|
|
535
|
+
}
|
|
536
|
+
|
|
480
537
|
function syncCodex(context) {
|
|
481
538
|
if (!context.commandAvailable('codex')) return { host: 'codex', status: 'skipped', reasonCode: 'cli-not-found', reason: 'codex CLI not found' };
|
|
482
539
|
const marketplace = ensureCodexMarketplace(context);
|
|
@@ -527,7 +584,19 @@ function syncCodex(context) {
|
|
|
527
584
|
|
|
528
585
|
const verification = verifyPlugin(context, 'Codex', queryCodexPlugin);
|
|
529
586
|
if (!verification.ok) return { host: 'codex', status: 'failed', error: verification.error };
|
|
530
|
-
|
|
587
|
+
const localSource = context.dryRun ? { ok: true, path: null } : codexLocalPluginRoot(verification.state && verification.state.plugin);
|
|
588
|
+
if (!localSource.ok) {
|
|
589
|
+
return {
|
|
590
|
+
host: 'codex',
|
|
591
|
+
status: 'failed',
|
|
592
|
+
action,
|
|
593
|
+
error: `Codex plugin ${action} succeeded but compatibility hooks could not be refreshed: ${localSource.error}`
|
|
594
|
+
};
|
|
595
|
+
}
|
|
596
|
+
const sourcePluginRoot = localSource.path;
|
|
597
|
+
const compatHooks = refreshCodexCompatibilityHooks(context, sourcePluginRoot);
|
|
598
|
+
if (!compatHooks.ok) return { host: 'codex', status: 'failed', action, error: 'Codex plugin ' + action + ' succeeded but compatibility hooks could not be refreshed: ' + compatHooks.error };
|
|
599
|
+
return { host: 'codex', status: 'synchronized', action, compatHooks: compatHooks.result };
|
|
531
600
|
}
|
|
532
601
|
|
|
533
602
|
function parseArgs(argv) {
|
|
@@ -560,7 +629,9 @@ function help() {
|
|
|
560
629
|
|
|
561
630
|
Synchronize the native Harness Everything plugin on installed host CLIs.
|
|
562
631
|
Existing installations take the explicit update/upgrade path; absent
|
|
563
|
-
installations take the install path.
|
|
632
|
+
installations take the install path. Successful Codex sync also refreshes the
|
|
633
|
+
local compatibility-hook runtime and user hook entries; Codex trust remains a
|
|
634
|
+
separate host review.
|
|
564
635
|
|
|
565
636
|
Options:
|
|
566
637
|
--host <all|claude|codex> Limit synchronization to one host (default: all)
|
|
@@ -615,7 +686,7 @@ function main(argv = process.argv.slice(2)) {
|
|
|
615
686
|
if (options.json) console.log(JSON.stringify(summary, null, 2));
|
|
616
687
|
else {
|
|
617
688
|
for (const result of summary.results) {
|
|
618
|
-
if (result.status === 'synchronized') console.log(
|
|
689
|
+
if (result.status === 'synchronized') console.log('[ok] ' + result.host + ': ' + result.action + (result.compatHooks ? '; Codex hooks ' + result.compatHooks.status : ''));
|
|
619
690
|
else if (result.status === 'skipped') console.log(`[skip] ${result.host}: ${result.reason}`);
|
|
620
691
|
else console.error(`[fail] ${result.host}: ${result.error}`);
|
|
621
692
|
}
|
|
@@ -14,6 +14,14 @@ function option(args, name, fallback) {
|
|
|
14
14
|
return i !== -1 ? args[i + 1] : fallback;
|
|
15
15
|
}
|
|
16
16
|
|
|
17
|
+
function parseSince(value) {
|
|
18
|
+
const text = String(value || '');
|
|
19
|
+
const normalized = /^\d{4}-\d{2}-\d{2}$/.test(text) ? text + 'T00:00:00.000Z' : text;
|
|
20
|
+
const timestamp = Date.parse(normalized);
|
|
21
|
+
if (!Number.isFinite(timestamp)) throw new Error('--since must be an ISO date or timestamp');
|
|
22
|
+
return timestamp;
|
|
23
|
+
}
|
|
24
|
+
|
|
17
25
|
function family(text) {
|
|
18
26
|
const normalized = String(text || '').toLowerCase().replace(/[\p{P}\p{S}\d]+/gu, ' ').replace(/\s+/g, ' ').trim().slice(0, 48);
|
|
19
27
|
return crypto.createHash('sha256').update(normalized).digest('hex').slice(0, 12);
|
|
@@ -37,6 +45,7 @@ function loadReview(file) {
|
|
|
37
45
|
|
|
38
46
|
function derive(record, owner) {
|
|
39
47
|
const self = record.selfReport;
|
|
48
|
+
const behavior = record.captureMethod === 'session-backfill' ? null : record.behavior;
|
|
40
49
|
let validity = self ? self.validity : null;
|
|
41
50
|
let contextDependent = self ? self.contextDependent : null;
|
|
42
51
|
if (owner && owner.validity && owner.validity !== 'unsure') validity = owner.validity;
|
|
@@ -48,7 +57,9 @@ function derive(record, owner) {
|
|
|
48
57
|
return {
|
|
49
58
|
id: record.id, validity, contextDependent, tier, tierSource,
|
|
50
59
|
intents: self ? self.intents : [], workflow: (self && self.workflow) || (record.router && record.router.strategy) || null,
|
|
51
|
-
skills,
|
|
60
|
+
skills,
|
|
61
|
+
behaviorEvidence: behavior ? 'observed' : 'unobserved',
|
|
62
|
+
contradictions: obs.contradictions(self, behavior),
|
|
52
63
|
selfReportReason: record.selfReportReason || null, host: record.host,
|
|
53
64
|
};
|
|
54
65
|
}
|
|
@@ -63,13 +74,23 @@ function run(args) {
|
|
|
63
74
|
if (!out) throw new Error('--out is required');
|
|
64
75
|
const share = Number(option(args, '--validation-share', '0.15'));
|
|
65
76
|
const owner = loadReview(option(args, '--owner-review'));
|
|
77
|
+
const sinceText = option(args, '--since', null);
|
|
78
|
+
const since = sinceText ? parseSince(sinceText) : null;
|
|
79
|
+
const selfReportOnly = args.includes('--self-report-only');
|
|
66
80
|
obs.sweep(store);
|
|
67
81
|
const index = obs.readJson(path.join(store, 'observations-index.json'), null);
|
|
68
82
|
if (!index || index.schemaVersion !== 1 || !Array.isArray(index.records)) throw new Error(`no observation index in ${store}`);
|
|
69
83
|
const rows = [];
|
|
70
84
|
let skippedNoText = 0;
|
|
85
|
+
let skippedBeforeSince = 0;
|
|
86
|
+
let skippedUnlabeled = 0;
|
|
71
87
|
const now = Date.now();
|
|
72
88
|
for (const record of index.records) {
|
|
89
|
+
if (since !== null) {
|
|
90
|
+
const observedAt = Date.parse(record.observedAt || '');
|
|
91
|
+
if (!Number.isFinite(observedAt) || observedAt < since) { skippedBeforeSince++; continue; }
|
|
92
|
+
}
|
|
93
|
+
if (selfReportOnly && !record.selfReport) { skippedUnlabeled++; continue; }
|
|
73
94
|
// Expiry holds on read too, even if no later turn has swept the store.
|
|
74
95
|
const text = obs.recordText(store, record, now);
|
|
75
96
|
if (!text || !text.prompt) { skippedNoText++; continue; }
|
|
@@ -91,9 +112,15 @@ function run(args) {
|
|
|
91
112
|
const count = key => rows.reduce((acc, r) => { const k = String(key(r)); acc[k] = (acc[k] || 0) + 1; return acc; }, {});
|
|
92
113
|
const report = {
|
|
93
114
|
schemaVersion: 1, exportedAt: new Date().toISOString(), records: index.records.length, exported: rows.length,
|
|
115
|
+
since: sinceText, selfReportOnly, skippedBeforeSince, skippedUnlabeled,
|
|
94
116
|
skippedNoText, validationRows: validation, withSelfReport: withSelf.length,
|
|
95
117
|
selfReportReasons: count(r => r.record.selfReportReason || 'ok'), hosts: count(r => r.record.host),
|
|
96
118
|
validity: count(r => r.label.validity), tiers: count(r => r.label.tier), tierSources: count(r => r.label.tierSource),
|
|
119
|
+
behaviorEvidence: {
|
|
120
|
+
observed: rows.filter(r => r.label.behaviorEvidence === 'observed').length,
|
|
121
|
+
unobserved: rows.filter(r => r.label.behaviorEvidence === 'unobserved').length,
|
|
122
|
+
},
|
|
123
|
+
contradictionEvidence: 'observed-behavior-only',
|
|
97
124
|
ownerDecisions: Object.keys(owner).length, agreement, contradictions,
|
|
98
125
|
};
|
|
99
126
|
fs.writeFileSync(path.join(out, 'export-report.json'), JSON.stringify(report, null, 2) + '\n');
|
|
@@ -103,4 +130,4 @@ function run(args) {
|
|
|
103
130
|
if (require.main === module) {
|
|
104
131
|
try { console.log(JSON.stringify(run(process.argv.slice(2)), null, 2)); } catch (error) { console.error(`ERROR: ${error.message}`); process.exitCode = 1; }
|
|
105
132
|
}
|
|
106
|
-
module.exports = { run, derive, family };
|
|
133
|
+
module.exports = { run, derive, family, parseSince };
|
|
@@ -0,0 +1,307 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
'use strict';
|
|
3
|
+
|
|
4
|
+
// Backfill valid Codex self-report labels from local rollout files into the
|
|
5
|
+
// private System One observations store. The importer never uploads session
|
|
6
|
+
// data and only persists user prompt text after the standard local redaction.
|
|
7
|
+
|
|
8
|
+
const fs = require('fs');
|
|
9
|
+
const os = require('os');
|
|
10
|
+
const path = require('path');
|
|
11
|
+
const readline = require('readline');
|
|
12
|
+
const { createReadStream } = require('fs');
|
|
13
|
+
const obs = require('../hooks/scripts/lib/observations');
|
|
14
|
+
|
|
15
|
+
const DEFAULT_SINCE = '2026-09-27';
|
|
16
|
+
|
|
17
|
+
function option(args, name, fallback) {
|
|
18
|
+
const index = args.indexOf(name);
|
|
19
|
+
return index === -1 ? fallback : args[index + 1];
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
function defaultCodexHome(env = process.env) {
|
|
23
|
+
return path.resolve(env.CODEX_HOME || path.join(os.homedir(), '.codex'));
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
function parseSince(value) {
|
|
27
|
+
const text = String(value || DEFAULT_SINCE);
|
|
28
|
+
const normalized = /^\d{4}-\d{2}-\d{2}$/.test(text) ? text + 'T00:00:00.000Z' : text;
|
|
29
|
+
const timestamp = Date.parse(normalized);
|
|
30
|
+
if (!Number.isFinite(timestamp)) throw new Error('--since must be an ISO date or timestamp');
|
|
31
|
+
return timestamp;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function sessionFiles(root) {
|
|
35
|
+
if (!fs.existsSync(root)) throw new Error('Codex sessions directory does not exist: ' + root);
|
|
36
|
+
const files = [];
|
|
37
|
+
function walk(directory) {
|
|
38
|
+
for (const entry of fs.readdirSync(directory, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {
|
|
39
|
+
const full = path.join(directory, entry.name);
|
|
40
|
+
if (entry.isDirectory()) walk(full);
|
|
41
|
+
else if (entry.isFile() && entry.name.endsWith('.jsonl')) files.push(full);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
walk(root);
|
|
45
|
+
return files;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function firstUserText(payload) {
|
|
49
|
+
if (typeof payload.content === 'string') return payload.content;
|
|
50
|
+
if (!Array.isArray(payload.content)) return '';
|
|
51
|
+
const item = payload.content.find(entry => entry && entry.type === 'input_text' && typeof entry.text === 'string');
|
|
52
|
+
return item ? item.text : '';
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
function assistantText(payload) {
|
|
56
|
+
if (typeof payload.content === 'string') return payload.content;
|
|
57
|
+
if (!Array.isArray(payload.content)) return '';
|
|
58
|
+
return payload.content
|
|
59
|
+
.filter(entry => entry && ['output_text', 'text'].includes(entry.type) && typeof entry.text === 'string')
|
|
60
|
+
.map(entry => entry.text)
|
|
61
|
+
.join('\n');
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function isoTimestamp(value) {
|
|
65
|
+
const timestamp = Date.parse(String(value || ''));
|
|
66
|
+
return Number.isFinite(timestamp) ? new Date(timestamp).toISOString() : null;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function completionLabel(message) {
|
|
70
|
+
const result = obs.parseLabelLine(typeof message === 'string' ? message : '');
|
|
71
|
+
if (result.label) return { label: result.label, reason: null };
|
|
72
|
+
return { label: null, reason: result.reason };
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
function reconcileFinalMessage(payload, current) {
|
|
76
|
+
const hasCompletionMessage = Object.prototype.hasOwnProperty.call(payload, 'last_agent_message') &&
|
|
77
|
+
typeof payload.last_agent_message === 'string';
|
|
78
|
+
const assistantCanRepresentFinal = current.lastAssistantPhase !== 'commentary';
|
|
79
|
+
const assistantMessage = assistantCanRepresentFinal ? current.lastAssistantMessage : '';
|
|
80
|
+
let message = hasCompletionMessage ? payload.last_agent_message : assistantMessage;
|
|
81
|
+
let parsed = completionLabel(message);
|
|
82
|
+
let ambiguous = false;
|
|
83
|
+
|
|
84
|
+
if (hasCompletionMessage && parsed.reason === 'label-missing' && assistantMessage) {
|
|
85
|
+
const assistantParsed = completionLabel(assistantMessage);
|
|
86
|
+
if (assistantParsed.label) {
|
|
87
|
+
const sameBody = obs.stripLabels(message) === obs.stripLabels(assistantMessage);
|
|
88
|
+
if (sameBody) {
|
|
89
|
+
message = assistantMessage;
|
|
90
|
+
parsed = assistantParsed;
|
|
91
|
+
} else if (!sameBody) {
|
|
92
|
+
// Both sources describe the final reply, but disagree about its body.
|
|
93
|
+
// Do not guess which text is authoritative or let either seed context.
|
|
94
|
+
message = null;
|
|
95
|
+
parsed = { label: null, reason: 'label-missing' };
|
|
96
|
+
ambiguous = true;
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
return { message, parsed, ambiguous };
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
function blankReport(filesScanned) {
|
|
105
|
+
return {
|
|
106
|
+
since: DEFAULT_SINCE,
|
|
107
|
+
filesScanned,
|
|
108
|
+
turnsScanned: 0,
|
|
109
|
+
eligibleTurns: 0,
|
|
110
|
+
validLabels: 0,
|
|
111
|
+
imported: 0,
|
|
112
|
+
migratedUnobservedBehavior: 0,
|
|
113
|
+
skipped: {
|
|
114
|
+
beforeCutoff: 0,
|
|
115
|
+
missingLabel: 0,
|
|
116
|
+
invalidLabel: 0,
|
|
117
|
+
missingPrompt: 0,
|
|
118
|
+
invalidTimestamp: 0,
|
|
119
|
+
turnMismatch: 0,
|
|
120
|
+
duplicate: 0,
|
|
121
|
+
ambiguousLiveMatch: 0,
|
|
122
|
+
ambiguousFinalMessage: 0,
|
|
123
|
+
},
|
|
124
|
+
failed: { lockTimeout: 0 },
|
|
125
|
+
errors: [],
|
|
126
|
+
};
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
async function importSessions(options = {}) {
|
|
130
|
+
const sessionsRoot = path.resolve(options.sessionsDir || path.join(defaultCodexHome(options.env), 'sessions'));
|
|
131
|
+
const store = path.resolve(options.store || obs.storeRoot(options.env));
|
|
132
|
+
const since = options.since || DEFAULT_SINCE;
|
|
133
|
+
const cutoff = parseSince(since);
|
|
134
|
+
const files = sessionFiles(sessionsRoot);
|
|
135
|
+
const report = blankReport(files.length);
|
|
136
|
+
report.since = since;
|
|
137
|
+
const migrated = obs.markSessionBackfillBehaviorUnobserved(store, { lockTimeoutMs: options.lockTimeoutMs });
|
|
138
|
+
if (migrated === null) {
|
|
139
|
+
report.failed.lockTimeout++;
|
|
140
|
+
report.errors.push('Observation store lock timed out during behavior migration; no session rows were imported.');
|
|
141
|
+
return report;
|
|
142
|
+
}
|
|
143
|
+
report.migratedUnobservedBehavior = migrated;
|
|
144
|
+
|
|
145
|
+
for (const file of files) {
|
|
146
|
+
let sessionId = path.basename(file, '.jsonl');
|
|
147
|
+
let turnNumber = 0;
|
|
148
|
+
let current = null;
|
|
149
|
+
let previousFinal = '';
|
|
150
|
+
let previousAt = null;
|
|
151
|
+
const lines = readline.createInterface({ input: createReadStream(file, { encoding: 'utf8' }), crlfDelay: Infinity });
|
|
152
|
+
|
|
153
|
+
for await (const line of lines) {
|
|
154
|
+
if (!line) continue;
|
|
155
|
+
let row;
|
|
156
|
+
try { row = JSON.parse(line); } catch (_) { continue; }
|
|
157
|
+
const payload = row.payload && typeof row.payload === 'object' ? row.payload : {};
|
|
158
|
+
if (row.type === 'session_meta') {
|
|
159
|
+
sessionId = String(payload.session_id || payload.id || sessionId);
|
|
160
|
+
continue;
|
|
161
|
+
}
|
|
162
|
+
if (row.type === 'event_msg' && payload.type === 'task_started') {
|
|
163
|
+
if (current) current = null;
|
|
164
|
+
turnNumber++;
|
|
165
|
+
current = {
|
|
166
|
+
turn: turnNumber,
|
|
167
|
+
turnId: String(payload.turn_id || ''),
|
|
168
|
+
startedAt: isoTimestamp(row.timestamp || payload.started_at),
|
|
169
|
+
prompt: '',
|
|
170
|
+
promptSeen: false,
|
|
171
|
+
lastAssistantMessage: '',
|
|
172
|
+
lastAssistantPhase: null,
|
|
173
|
+
previous: '',
|
|
174
|
+
previousAt: null,
|
|
175
|
+
};
|
|
176
|
+
if (previousFinal && previousAt && current.startedAt) {
|
|
177
|
+
const age = Date.parse(current.startedAt) - Date.parse(previousAt);
|
|
178
|
+
if (age >= 0 && age <= obs.RETENTION_MS) {
|
|
179
|
+
current.previous = obs.redact(obs.tailBytes(obs.stripLabels(previousFinal), obs.PREVIOUS_BYTES));
|
|
180
|
+
current.previousAt = previousAt;
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
continue;
|
|
184
|
+
}
|
|
185
|
+
if (!current) continue;
|
|
186
|
+
if (row.type === 'response_item' && payload.type === 'message' && payload.role === 'user' && !current.promptSeen) {
|
|
187
|
+
current.promptSeen = true;
|
|
188
|
+
current.prompt = firstUserText(payload);
|
|
189
|
+
continue;
|
|
190
|
+
}
|
|
191
|
+
if (row.type === 'response_item' && payload.type === 'message' && payload.role === 'assistant') {
|
|
192
|
+
current.lastAssistantMessage = assistantText(payload);
|
|
193
|
+
current.lastAssistantPhase = typeof payload.phase === 'string' ? payload.phase : null;
|
|
194
|
+
continue;
|
|
195
|
+
}
|
|
196
|
+
if (row.type !== 'event_msg' || payload.type !== 'task_complete') continue;
|
|
197
|
+
const completedTurnId = String(payload.turn_id || '');
|
|
198
|
+
if (current.turnId && completedTurnId && current.turnId !== completedTurnId) {
|
|
199
|
+
report.skipped.turnMismatch++;
|
|
200
|
+
continue;
|
|
201
|
+
}
|
|
202
|
+
if (!current.turnId && completedTurnId) current.turnId = completedTurnId;
|
|
203
|
+
|
|
204
|
+
report.turnsScanned++;
|
|
205
|
+
const startedMs = Date.parse(current.startedAt || '');
|
|
206
|
+
const reconciledFinal = reconcileFinalMessage(payload, current);
|
|
207
|
+
const finalMessage = reconciledFinal.message;
|
|
208
|
+
if (!Number.isFinite(startedMs)) {
|
|
209
|
+
report.skipped.invalidTimestamp++;
|
|
210
|
+
} else if (startedMs < cutoff) {
|
|
211
|
+
report.skipped.beforeCutoff++;
|
|
212
|
+
} else {
|
|
213
|
+
report.eligibleTurns++;
|
|
214
|
+
const parsed = reconciledFinal.parsed;
|
|
215
|
+
if (reconciledFinal.ambiguous) {
|
|
216
|
+
report.skipped.ambiguousFinalMessage++;
|
|
217
|
+
} else if (!parsed.label) {
|
|
218
|
+
report.skipped[parsed.reason === 'label-missing' ? 'missingLabel' : 'invalidLabel']++;
|
|
219
|
+
} else {
|
|
220
|
+
report.validLabels++;
|
|
221
|
+
if (!current.prompt || !current.prompt.trim()) {
|
|
222
|
+
report.skipped.missingPrompt++;
|
|
223
|
+
} else {
|
|
224
|
+
const prompt = obs.redact(current.prompt);
|
|
225
|
+
const previous = current.previous || '';
|
|
226
|
+
const text = { prompt, previous };
|
|
227
|
+
const completedAt = isoTimestamp(payload.completed_at || row.timestamp) || current.startedAt;
|
|
228
|
+
const capturedAt = current.previousAt
|
|
229
|
+
? Math.min(startedMs, Date.parse(current.previousAt))
|
|
230
|
+
: startedMs;
|
|
231
|
+
const validUntil = new Date(capturedAt + obs.RETENTION_MS).toISOString();
|
|
232
|
+
const expired = Date.parse(validUntil) <= Date.now();
|
|
233
|
+
const record = {
|
|
234
|
+
id: obs.sha('session-backfill:' + sessionId + ':' + (current.turnId || current.turn) + ':' + current.startedAt, 24),
|
|
235
|
+
source: 'observation',
|
|
236
|
+
captureMethod: 'session-backfill',
|
|
237
|
+
status: expired ? 'expired' : 'active',
|
|
238
|
+
...(expired ? { textDeleted: true } : {}),
|
|
239
|
+
observedAt: completedAt,
|
|
240
|
+
validUntil,
|
|
241
|
+
contentSha256: obs.sha(JSON.stringify(text)),
|
|
242
|
+
host: 'codex',
|
|
243
|
+
turn: current.turn,
|
|
244
|
+
writer: { sessionId, ...(current.turnId ? { turnId: current.turnId } : {}) },
|
|
245
|
+
scope: { taskTerms: obs.normalizeTerms(prompt), requirementTerms: [], roles: [] },
|
|
246
|
+
promptBytes: Buffer.byteLength(prompt, 'utf8'),
|
|
247
|
+
router: null,
|
|
248
|
+
behavior: null,
|
|
249
|
+
selfReport: parsed.label,
|
|
250
|
+
selfReportReason: null,
|
|
251
|
+
};
|
|
252
|
+
const appendOptions = {
|
|
253
|
+
dedupeById: true,
|
|
254
|
+
dedupeByTurn: true,
|
|
255
|
+
...(options.lockTimeoutMs === undefined ? {} : { lockTimeoutMs: options.lockTimeoutMs }),
|
|
256
|
+
};
|
|
257
|
+
const appended = obs.appendRecordDetailed(store, record, expired ? null : text, appendOptions);
|
|
258
|
+
if (appended.status === 'appended') report.imported++;
|
|
259
|
+
else if (appended.status === 'duplicate') report.skipped.duplicate++;
|
|
260
|
+
else if (appended.status === 'ambiguous') report.skipped.ambiguousLiveMatch++;
|
|
261
|
+
else if (appended.status === 'lock-timeout') {
|
|
262
|
+
report.failed.lockTimeout++;
|
|
263
|
+
report.errors.push('Observation store lock timed out while appending a session turn; the record was not persisted.');
|
|
264
|
+
return report;
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
previousFinal = reconciledFinal.ambiguous ? '' : (typeof finalMessage === 'string' ? finalMessage : '');
|
|
271
|
+
previousAt = previousFinal ? isoTimestamp(payload.completed_at || row.timestamp) : null;
|
|
272
|
+
current = null;
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
return report;
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
function parseArgs(args) {
|
|
279
|
+
if (args.length % 2 !== 0) throw new Error('Usage: node scripts/system-one-observations-import-codex-sessions.js [--sessions DIR] [--store DIR] [--since YYYY-MM-DD]');
|
|
280
|
+
const result = { sessionsDir: undefined, store: undefined, since: DEFAULT_SINCE };
|
|
281
|
+
for (let index = 0; index < args.length; index += 2) {
|
|
282
|
+
const name = args[index];
|
|
283
|
+
const value = args[index + 1];
|
|
284
|
+
if (!value || !['--sessions', '--store', '--since'].includes(name)) {
|
|
285
|
+
throw new Error('Usage: node scripts/system-one-observations-import-codex-sessions.js [--sessions DIR] [--store DIR] [--since YYYY-MM-DD]');
|
|
286
|
+
}
|
|
287
|
+
if (name === '--sessions') result.sessionsDir = value;
|
|
288
|
+
else if (name === '--store') result.store = value;
|
|
289
|
+
else result.since = value;
|
|
290
|
+
}
|
|
291
|
+
return result;
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
async function main(args = process.argv.slice(2)) {
|
|
295
|
+
try {
|
|
296
|
+
const report = await importSessions(parseArgs(args));
|
|
297
|
+
process.stdout.write(JSON.stringify(report, null, 2) + '\n');
|
|
298
|
+
return report.failed.lockTimeout ? 1 : 0;
|
|
299
|
+
} catch (error) {
|
|
300
|
+
process.stderr.write('ERROR: ' + error.message + '\n');
|
|
301
|
+
return 1;
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
if (require.main === module) main().then(code => { process.exitCode = code; });
|
|
306
|
+
|
|
307
|
+
module.exports = { DEFAULT_SINCE, assistantText, firstUserText, importSessions, parseSince, reconcileFinalMessage, sessionFiles };
|
|
@@ -21,6 +21,8 @@ function option(args, name, fallback) {
|
|
|
21
21
|
function disagrees(record) {
|
|
22
22
|
const self = record.selfReport;
|
|
23
23
|
if (!self) return true;
|
|
24
|
+
const behaviorObserved = record.captureMethod !== 'session-backfill' && record.behavior !== null && record.behavior !== undefined;
|
|
25
|
+
if (!behaviorObserved) return true;
|
|
24
26
|
if (obs.contradictions(self, record.behavior).length) return true;
|
|
25
27
|
return Boolean(record.router && record.router.tier && self.tier && record.router.tier !== self.tier);
|
|
26
28
|
}
|
|
@@ -39,6 +41,7 @@ main{max-width:860px;margin:0 auto;padding:16px}.box{white-space:pre-wrap;word-b
|
|
|
39
41
|
<div class="row"><span id="pos"></span><button id="prev">←</button><button id="next">→</button><button id="exp">Export JSON</button><span id="stat" class="prev"></span></div>
|
|
40
42
|
<h3>Previous assistant message</h3><div class="box prev" id="previous"></div>
|
|
41
43
|
<h3>Prompt</h3><div class="box" id="prompt"></div>
|
|
44
|
+
<h3>Behavior evidence</h3><div class="box" id="behavior"></div>
|
|
42
45
|
<div class="row"><label>Validity <select id="validity"><option value="">—</option><option>actionable</option><option>invalid</option><option>unsure</option></select></label>
|
|
43
46
|
<label>Needs context <select id="contextDependent"><option value="">—</option><option value="true">yes</option><option value="false">no</option></select></label>
|
|
44
47
|
<label>Tier <select id="tier"><option value="">—</option><option>tier1</option><option>tier2</option><option>tier3</option></select></label></div>
|
|
@@ -49,6 +52,7 @@ const $=id=>document.getElementById(id);
|
|
|
49
52
|
function save(){try{localStorage.setItem(KEY,JSON.stringify(st))}catch(e){}}
|
|
50
53
|
function render(){if(!ITEMS.length){$('prompt').textContent='Nothing to review.';return}const it=ITEMS[i],d=st[it.id]||{};
|
|
51
54
|
$('pos').textContent=(i+1)+' / '+ITEMS.length;$('prompt').textContent=it.prompt;$('previous').textContent=it.previous||'(none)';
|
|
55
|
+
$('behavior').textContent=it.behaviorEvidence==='observed'?'Observed by local hook collection.':'Behavior evidence is unobserved; session backfill did not record tool events or file writes.';
|
|
52
56
|
$('validity').value=d.validity||'';$('tier').value=d.tier||'';$('contextDependent').value=typeof d.contextDependent==='boolean'?String(d.contextDependent):'';$('stat').textContent='decided '+Object.keys(st).length}
|
|
53
57
|
function set(){const it=ITEMS[i];const cd=$('contextDependent').value;st[it.id]={id:it.id,validity:$('validity').value||null,contextDependent:cd==='true'?true:cd==='false'?false:null,tier:$('tier').value||null,decidedAt:new Date().toISOString()};save();render()}
|
|
54
58
|
$('validity').onchange=set;$('contextDependent').onchange=set;$('tier').onchange=set;$('prev').onclick=()=>{if(i>0)i--;render()};$('next').onclick=()=>{if(i<ITEMS.length-1)i++;render()};
|
|
@@ -72,7 +76,13 @@ function run(args) {
|
|
|
72
76
|
if (!all && !disagrees(record)) continue;
|
|
73
77
|
const text = obs.recordText(store, record, now);
|
|
74
78
|
if (!text || !text.prompt) continue;
|
|
75
|
-
items.push({
|
|
79
|
+
items.push({
|
|
80
|
+
id: record.id,
|
|
81
|
+
prompt: text.prompt,
|
|
82
|
+
previous: text.previous || '',
|
|
83
|
+
behaviorEvidence: record.captureMethod !== 'session-backfill' && record.behavior !== null && record.behavior !== undefined
|
|
84
|
+
? 'observed' : 'unobserved',
|
|
85
|
+
});
|
|
76
86
|
if (items.length >= limit) break;
|
|
77
87
|
}
|
|
78
88
|
fs.mkdirSync(path.dirname(path.resolve(out)), { recursive: true });
|