codemem 0.36.0 → 0.37.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +587 -394
- package/dist/index.js.map +1 -1
- package/package.json +5 -5
package/dist/index.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import { DEDUP_KEY_BACKFILL_JOB, DEFAULT_COORDINATOR_DB_PATH, DedupKeyBackfillRunner, MUTATING_TOOL_NAMES, MemoryStore, ObserverClient, REF_BACKFILL_JOB, RawEventSweeper, RefBackfillRunner, SCOPE_BACKFILL_JOB, SESSION_CONTEXT_BACKFILL_JOB, SUMMARY_DEDUP_BACKFILL_JOB, ScopeBackfillRunner, SessionContextBackfillRunner, SummaryDedupBackfillRunner, SyncRetentionRunner, VERSION, VectorModelMigrationRunner, aiBackfillStructuredContent, applyBootstrapSnapshot, backfillMemoryDedupKeys, backfillNarrativeFromBody, backfillTagsText, backfillVectors, buildAuthHeaders, buildBaseUrl, buildRawEventEnvelopeFromCodexHook, buildRawEventEnvelopeFromHook, compareMemoryRoleReports, connect, coordinatorCreateGroupAction, coordinatorCreateInviteAction, coordinatorCreateScopeAction, coordinatorDisableDeviceAction, coordinatorEnrollDeviceAction, coordinatorGrantScopeMembershipAction, coordinatorImportInviteAction, coordinatorListBootstrapGrantsAction, coordinatorListDevicesAction, coordinatorListGroupsAction, coordinatorListJoinRequestsAction, coordinatorListScopeMembershipsAction, coordinatorListScopesAction, coordinatorRemoveDeviceAction, coordinatorRenameDeviceAction, coordinatorReviewJoinRequestAction, coordinatorRevokeBootstrapGrantAction, coordinatorRevokeScopeMembershipAction, coordinatorUpdateScopeAction, createBetterSqliteCoordinatorApp, deactivateLowSignalMemories, deactivateLowSignalObservations, dedupNearDuplicateMemories, ensureDeviceIdentity, ensureSchemaBootstrapped, exportMemories, extractApplyPatchPaths, fetchAllSnapshotPages, fingerprintPublicKey, flushRawEvents, formatHostPort, getExtractionBenchmarkProfile, getInjectionEvalScenarioPack, getInjectionEvalScenarioPrompts, getMaintenanceJob, getMemoryRoleReport, getRawEventRelinkPlan, getRawEventRelinkReport, getRawEventStatus, getSemanticIndexDiagnostics, getSessionExtractionEval, getSessionExtractionEvalScenario, getWorkspaceCodememConfigPath, hasPendingDedupKeyBackfill, hasPendingRefBackfill, hasPendingScopeBackfill, hasPendingSessionContextBackfill, hasPendingSummaryDedupBackfill, hasUnsyncedSharedMemoryChanges, importMemories, initDatabase, isEmbeddingDisabled, listMaintenanceJobs, listPerPeerScopeSyncState, loadObserverConfig, loadPublicKey, loadSqliteVec, mdnsEnabled, planReplicationOpsAgePrune, pruneReplicationOpsUntilCaughtUp, rawEventsGate, readCodememConfigFile, readCodememConfigFileAtPath, readCoordinatorSyncConfig, readImportPayload, replayBatchExtraction, replayBatchExtractionWithTierRouting, requestJson, resolveCodememConfigPath, resolveDbPath, resolveHookProject, resolveProject, retryRawEventFailures, runSyncDaemon, runSyncPass, scanSecretsRetroactive, schema, setPeerProjectFilter, stripJsonComments, stripPrivateObj, stripTrailingCommas, syncPassPreflight, updatePeerAddresses, vacuumDatabase, writeCodememConfigFile } from "@codemem/core";
|
|
2
|
+
import { DEDUP_KEY_BACKFILL_JOB, DEFAULT_COORDINATOR_DB_PATH, DedupKeyBackfillRunner, MUTATING_TOOL_NAMES, MemoryStore, ObserverClient, REF_BACKFILL_JOB, RawEventSweeper, RefBackfillRunner, SCOPE_BACKFILL_JOB, SESSION_CONTEXT_BACKFILL_JOB, SUMMARY_DEDUP_BACKFILL_JOB, ScopeBackfillRunner, SessionContextBackfillRunner, SummaryDedupBackfillRunner, SyncRetentionRunner, VERSION, VectorModelMigrationRunner, aiBackfillStructuredContent, applyBootstrapSnapshot, backfillMemoryDedupKeys, backfillNarrativeFromBody, backfillTagsText, backfillVectors, buildAuthHeaders, buildBaseUrl, buildRawEventEnvelopeFromCodexHook, buildRawEventEnvelopeFromHook, compareMemoryRoleReports, connect, coordinatorCreateGroupAction, coordinatorCreateInviteAction, coordinatorCreateScopeAction, coordinatorDisableDeviceAction, coordinatorEnrollDeviceAction, coordinatorGrantScopeMembershipAction, coordinatorImportInviteAction, coordinatorListBootstrapGrantsAction, coordinatorListDevicesAction, coordinatorListGroupsAction, coordinatorListJoinRequestsAction, coordinatorListScopeMembershipsAction, coordinatorListScopesAction, coordinatorRemoveDeviceAction, coordinatorRenameDeviceAction, coordinatorReviewJoinRequestAction, coordinatorRevokeBootstrapGrantAction, coordinatorRevokeScopeMembershipAction, coordinatorUpdateScopeAction, createBetterSqliteCoordinatorApp, deactivateLowSignalMemories, deactivateLowSignalObservations, dedupNearDuplicateMemories, ensureDeviceIdentity, ensureSchemaBootstrapped, exportMemories, extractApplyPatchPaths, fetchAllSnapshotPages, fingerprintPublicKey, flushRawEvents, formatHostPort, getExtractionBenchmarkProfile, getInjectionEvalScenarioPack, getInjectionEvalScenarioPrompts, getMaintenanceJob, getMemoryRoleReport, getRawEventRelinkPlan, getRawEventRelinkReport, getRawEventStatus, getSemanticIndexDiagnostics, getSessionExtractionEval, getSessionExtractionEvalScenario, getWorkspaceCodememConfigPath, hasPendingDedupKeyBackfill, hasPendingRefBackfill, hasPendingScopeBackfill, hasPendingSessionContextBackfill, hasPendingSummaryDedupBackfill, hasUnsyncedSharedMemoryChanges, importMemories, initDatabase, isEmbeddingDisabled, listMaintenanceJobs, listPerPeerScopeSyncState, listRetentionScopeIds, loadObserverConfig, loadPublicKey, loadSqliteVec, mdnsEnabled, planReplicationOpsAgePrune, pruneReplicationOpsUntilCaughtUp, rawEventsGate, readCodememConfigFile, readCodememConfigFileAtPath, readCoordinatorSyncConfig, readImportPayload, replayBatchExtraction, replayBatchExtractionWithTierRouting, requestJson, resolveCodememConfigPath, resolveDbPath, resolveHookProject, resolveProject, retryRawEventFailures, runSyncDaemon, runSyncPass, scanSecretsRetroactive, schema, setPeerProjectFilter, stripJsonComments, stripPrivateObj, stripTrailingCommas, syncPassPreflight, updatePeerAddresses, vacuumDatabase, writeCodememConfigFile } from "@codemem/core";
|
|
3
3
|
import { Command, Option } from "commander";
|
|
4
4
|
import omelette from "omelette";
|
|
5
5
|
import { appendFileSync, copyFileSync, existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, rmdirSync, statSync, unlinkSync, writeFileSync } from "node:fs";
|
|
@@ -2864,18 +2864,33 @@ function buildCoordinatorCommand() {
|
|
|
2864
2864
|
coordServeCmd.addOption(new Option("--host <host>", "bind host").hideHelp());
|
|
2865
2865
|
coordServeCmd.addOption(new Option("--port <port>", "bind port").hideHelp());
|
|
2866
2866
|
coordServeCmd.action(async (opts) => {
|
|
2867
|
-
|
|
2868
|
-
|
|
2869
|
-
|
|
2870
|
-
|
|
2871
|
-
|
|
2872
|
-
|
|
2873
|
-
|
|
2874
|
-
|
|
2875
|
-
|
|
2876
|
-
|
|
2877
|
-
|
|
2878
|
-
|
|
2867
|
+
try {
|
|
2868
|
+
const host = String(opts.coordinatorHost ?? opts.host ?? "127.0.0.1").trim() || "127.0.0.1";
|
|
2869
|
+
const port = Number.parseInt(String(opts.coordinatorPort ?? opts.port ?? "7347"), 10);
|
|
2870
|
+
if (!Number.isFinite(port) || port < 0 || port > 65535) {
|
|
2871
|
+
p.log.error(`Invalid port: ${opts.coordinatorPort ?? opts.port}`);
|
|
2872
|
+
process.exitCode = 1;
|
|
2873
|
+
return;
|
|
2874
|
+
}
|
|
2875
|
+
const dbPath = resolveDbOpt(opts) ?? DEFAULT_COORDINATOR_DB_PATH;
|
|
2876
|
+
const app = createBetterSqliteCoordinatorApp({ dbPath });
|
|
2877
|
+
p.intro("codemem coordinator serve");
|
|
2878
|
+
serve({
|
|
2879
|
+
fetch: app.fetch,
|
|
2880
|
+
hostname: host,
|
|
2881
|
+
port
|
|
2882
|
+
}, (info) => {
|
|
2883
|
+
p.log.success(`Coordinator listening at http://${host}:${info.port}`);
|
|
2884
|
+
p.log.info(`DB: ${dbPath}`);
|
|
2885
|
+
}).on("error", (error) => {
|
|
2886
|
+
const message = error.code === "EADDRINUSE" ? `Port ${port} is already in use` : error instanceof Error ? error.message : String(error);
|
|
2887
|
+
p.log.error(message);
|
|
2888
|
+
process.exitCode = 1;
|
|
2889
|
+
});
|
|
2890
|
+
} catch (error) {
|
|
2891
|
+
p.log.error(error instanceof Error ? error.message : String(error));
|
|
2892
|
+
process.exitCode = 1;
|
|
2893
|
+
}
|
|
2879
2894
|
});
|
|
2880
2895
|
cmd.addCommand(coordServeCmd);
|
|
2881
2896
|
const listBootstrapGrantsCmd = new Command("list-bootstrap-grants").configureHelp(helpStyle).description("List bootstrap grants for a coordinator group").argument("<group>", "group id").option("--remote-url <url>", "remote coordinator URL override").option("--admin-secret <secret>", "remote coordinator admin secret override");
|
|
@@ -3137,22 +3152,32 @@ var dbCommand = new Command("db").configureHelp(helpStyle).description("Database
|
|
|
3137
3152
|
var initCmd = new Command("init").configureHelp(helpStyle).description("Create or verify the SQLite database and schema");
|
|
3138
3153
|
addDbOption(initCmd);
|
|
3139
3154
|
initCmd.action((opts) => {
|
|
3140
|
-
|
|
3141
|
-
|
|
3142
|
-
|
|
3143
|
-
|
|
3155
|
+
try {
|
|
3156
|
+
const result = initDatabase(resolveDbOpt(opts));
|
|
3157
|
+
p.intro("codemem db init");
|
|
3158
|
+
p.log.success(`Database ready: ${result.path}`);
|
|
3159
|
+
p.outro(`Size: ${result.sizeBytes.toLocaleString()} bytes`);
|
|
3160
|
+
} catch (error) {
|
|
3161
|
+
p.log.error(error instanceof Error ? error.message : String(error));
|
|
3162
|
+
process.exitCode = 1;
|
|
3163
|
+
}
|
|
3144
3164
|
});
|
|
3145
3165
|
dbCommand.addCommand(initCmd);
|
|
3146
3166
|
var vacuumCmd = new Command("vacuum").configureHelp(helpStyle).description("Run VACUUM on the SQLite database");
|
|
3147
3167
|
addDbOption(vacuumCmd);
|
|
3148
3168
|
vacuumCmd.action((opts) => {
|
|
3149
|
-
|
|
3150
|
-
|
|
3151
|
-
|
|
3152
|
-
|
|
3169
|
+
try {
|
|
3170
|
+
const result = vacuumDatabase(resolveDbOpt(opts));
|
|
3171
|
+
p.intro("codemem db vacuum");
|
|
3172
|
+
p.log.success(`Vacuumed: ${result.path}`);
|
|
3173
|
+
p.outro(`Size: ${result.sizeBytes.toLocaleString()} bytes`);
|
|
3174
|
+
} catch (error) {
|
|
3175
|
+
p.log.error(error instanceof Error ? error.message : String(error));
|
|
3176
|
+
process.exitCode = 1;
|
|
3177
|
+
}
|
|
3153
3178
|
});
|
|
3154
3179
|
dbCommand.addCommand(vacuumCmd);
|
|
3155
|
-
var pruneReplCmd = new Command("prune-replication-ops").configureHelp(helpStyle).description("Prune replication op history with approximate oldest-first retention, dry-run, and progress reporting").option("--dry-run", "show current size/targets without deleting").option("--max-age-days <days>", "retention age threshold in days", "30").option("--max-size-mb <mb>", "target replication log budget in MB", "512").option("--batch-ops <n>", "max ops deleted per batch", "5000").option("--batch-runtime-ms <ms>", "max runtime per batch in ms", "2000").option("--vacuum", "run VACUUM explicitly after prune completes");
|
|
3180
|
+
var pruneReplCmd = new Command("prune-replication-ops").configureHelp(helpStyle).description("Prune replication op history with approximate oldest-first retention, dry-run, and progress reporting").option("--dry-run", "show current size/targets without deleting").option("--max-age-days <days>", "retention age threshold in days", "30").option("--max-size-mb <mb>", "target per-scope replication log budget in MB", "512").option("--batch-ops <n>", "max ops deleted per batch", "5000").option("--batch-runtime-ms <ms>", "max runtime per batch in ms", "2000").option("--vacuum", "run VACUUM explicitly after prune completes");
|
|
3156
3181
|
addDbOption(pruneReplCmd);
|
|
3157
3182
|
pruneReplCmd.action((opts) => {
|
|
3158
3183
|
const dbPath = resolveDbPath(resolveDbOpt(opts));
|
|
@@ -3166,29 +3191,41 @@ pruneReplCmd.action((opts) => {
|
|
|
3166
3191
|
const beforeBytes = estimateReplicationOpsBytes(db);
|
|
3167
3192
|
p.intro("codemem db prune-replication-ops");
|
|
3168
3193
|
p.log.info(`Replication ops size: ${formatBytes(beforeBytes)}`);
|
|
3169
|
-
p.log.info(`Policy: approximately prune oldest-first toward <= ${maxSizeMb} MB while removing history older than ${maxAgeDays} day(s), subject to per-pass batch/runtime limits`);
|
|
3194
|
+
p.log.info(`Policy: across all replication scopes, approximately prune oldest-first toward <= ${maxSizeMb} MB PER SCOPE while removing history older than ${maxAgeDays} day(s), subject to per-pass batch/runtime limits (the size budget is applied per scope, matching the background retention runner; total ops across scopes may exceed it)`);
|
|
3195
|
+
const scopeIds = listRetentionScopeIds(db);
|
|
3170
3196
|
const agePlan = planReplicationOpsAgePrune(db, maxAgeDays, batchOps);
|
|
3171
|
-
if (agePlan.candidate_ops > 0) p.log.info(`Age pass plan: ${agePlan.candidate_ops.toLocaleString()} ops, ~${formatBytes(agePlan.estimated_candidate_bytes)} in ~${agePlan.estimated_batches.toLocaleString()} batch(es), cutoff ${agePlan.cutoff_cursor}`);
|
|
3197
|
+
if (agePlan.candidate_ops > 0) p.log.info(`Age pass plan (all scopes): ${agePlan.candidate_ops.toLocaleString()} ops, ~${formatBytes(agePlan.estimated_candidate_bytes)} in ~${agePlan.estimated_batches.toLocaleString()} batch(es), cutoff ${agePlan.cutoff_cursor}`);
|
|
3172
3198
|
else p.log.info("Age pass plan: no ops older than cutoff");
|
|
3173
3199
|
if (opts.dryRun) {
|
|
3174
3200
|
p.outro("Dry run only; no changes made");
|
|
3175
3201
|
return;
|
|
3176
3202
|
}
|
|
3177
|
-
|
|
3178
|
-
|
|
3179
|
-
|
|
3180
|
-
|
|
3181
|
-
|
|
3182
|
-
|
|
3183
|
-
|
|
3184
|
-
|
|
3185
|
-
|
|
3186
|
-
|
|
3187
|
-
|
|
3188
|
-
|
|
3189
|
-
|
|
3190
|
-
|
|
3191
|
-
|
|
3203
|
+
let totalDeleted = 0;
|
|
3204
|
+
let anyStoppedByBudget = false;
|
|
3205
|
+
let lastFloor = null;
|
|
3206
|
+
for (const scopeId of scopeIds) {
|
|
3207
|
+
const loopResult = pruneReplicationOpsUntilCaughtUp(db, {
|
|
3208
|
+
maxAgeDays,
|
|
3209
|
+
maxSizeBytes: maxSizeMb * 1024 * 1024,
|
|
3210
|
+
maxDeleteOps: batchOps,
|
|
3211
|
+
maxRuntimeMs: batchRuntimeMs,
|
|
3212
|
+
scopeId,
|
|
3213
|
+
onPass: (pass) => {
|
|
3214
|
+
if (pass.deleted === 0 && !pass.stoppedByBudget) return;
|
|
3215
|
+
const suffix = pass.stoppedByBudget ? " (budget-limited)" : "";
|
|
3216
|
+
p.log.step(`scope ${scopeId} pass ${pass.passNumber}: deleted ${pass.deleted.toLocaleString()} ops${suffix}`);
|
|
3217
|
+
}
|
|
3218
|
+
});
|
|
3219
|
+
totalDeleted += loopResult.totalDeleted;
|
|
3220
|
+
if (loopResult.totalDeleted > 0) p.log.info(`scope ${scopeId}: deleted ${loopResult.totalDeleted.toLocaleString()} ops`);
|
|
3221
|
+
if (loopResult.lastFloor) lastFloor = loopResult.lastFloor;
|
|
3222
|
+
if (loopResult.stoppedByBudget) anyStoppedByBudget = true;
|
|
3223
|
+
}
|
|
3224
|
+
p.log.info(`Deleted ops (all scopes): ${totalDeleted.toLocaleString()}`);
|
|
3225
|
+
const afterBytes = estimateReplicationOpsBytes(db);
|
|
3226
|
+
p.log.info(`Estimated replication ops size after prune: ${formatBytes(afterBytes)} (approximate)`);
|
|
3227
|
+
if (lastFloor) p.log.info(`Retained floor: ${lastFloor}`);
|
|
3228
|
+
if (anyStoppedByBudget) p.log.warn("Prune stopped because the current batch/runtime budget was exhausted before retention caught up. Re-run with higher --batch-ops or --batch-runtime-ms for faster catch-up.");
|
|
3192
3229
|
if (opts.vacuum) {
|
|
3193
3230
|
p.log.step("Running VACUUM as requested...");
|
|
3194
3231
|
db.close();
|
|
@@ -3203,31 +3240,88 @@ pruneReplCmd.action((opts) => {
|
|
|
3203
3240
|
}
|
|
3204
3241
|
});
|
|
3205
3242
|
dbCommand.addCommand(pruneReplCmd);
|
|
3243
|
+
var pruneRawEventsCmd = new Command("prune-raw-events").configureHelp(helpStyle).description("Delete raw_events older than a cutoff (age-based), with dry-run and VACUUM support").option("--dry-run", "show current size/targets without deleting").option("--max-age-days <days>", "retention age threshold in days", "90").option("--vacuum", "run VACUUM explicitly after prune completes");
|
|
3244
|
+
addDbOption(pruneRawEventsCmd);
|
|
3245
|
+
pruneRawEventsCmd.action((opts) => {
|
|
3246
|
+
const rawAge = opts.maxAgeDays.trim();
|
|
3247
|
+
const maxAgeDays = Number.parseInt(rawAge, 10);
|
|
3248
|
+
if (!/^\d+$/.test(rawAge) || !Number.isInteger(maxAgeDays) || maxAgeDays < 1) {
|
|
3249
|
+
p.log.error(`--max-age-days must be a positive integer (got "${opts.maxAgeDays}"). Refusing to run a destructive prune on invalid input.`);
|
|
3250
|
+
process.exitCode = 1;
|
|
3251
|
+
return;
|
|
3252
|
+
}
|
|
3253
|
+
const dbPath = resolveDbPath(resolveDbOpt(opts));
|
|
3254
|
+
let store = null;
|
|
3255
|
+
try {
|
|
3256
|
+
store = new MemoryStore(dbPath);
|
|
3257
|
+
const maxAgeMs = maxAgeDays * 864e5;
|
|
3258
|
+
const status = store.rawEventsRetentionStatus(maxAgeMs);
|
|
3259
|
+
p.intro("codemem db prune-raw-events");
|
|
3260
|
+
p.log.warn("raw_events is the re-extraction source. Age-based purge is NOT extraction-aware: pruning a window shorter than your extraction lag can delete un-extracted events. The 90-day default mitigates this.");
|
|
3261
|
+
p.log.info(`raw_events: ${status.total_rows.toLocaleString()} rows, ~${formatBytes(status.estimated_bytes)} on disk`);
|
|
3262
|
+
p.log.info(`Older than ${maxAgeDays} day(s): ${status.candidate_rows.toLocaleString()} row(s) to delete`);
|
|
3263
|
+
if (opts.dryRun) {
|
|
3264
|
+
p.outro("Dry run only; no changes made");
|
|
3265
|
+
return;
|
|
3266
|
+
}
|
|
3267
|
+
const deleted = store.purgeRawEvents(maxAgeMs);
|
|
3268
|
+
p.log.info(`Deleted raw_events: ${deleted.toLocaleString()}`);
|
|
3269
|
+
if (opts.vacuum) {
|
|
3270
|
+
p.log.step("Running VACUUM as requested...");
|
|
3271
|
+
store.close();
|
|
3272
|
+
store = null;
|
|
3273
|
+
const vacuumed = vacuumDatabase(dbPath);
|
|
3274
|
+
p.outro(`Done. VACUUM complete. File size is now ${formatBytes(vacuumed.sizeBytes)}.`);
|
|
3275
|
+
return;
|
|
3276
|
+
}
|
|
3277
|
+
p.outro("Done. SQLite file size may not shrink until you run `codemem db vacuum` explicitly (or re-run this command with --vacuum).");
|
|
3278
|
+
} catch (error) {
|
|
3279
|
+
p.log.error(error instanceof Error ? error.message : String(error));
|
|
3280
|
+
process.exitCode = 1;
|
|
3281
|
+
} finally {
|
|
3282
|
+
store?.close();
|
|
3283
|
+
}
|
|
3284
|
+
});
|
|
3285
|
+
dbCommand.addCommand(pruneRawEventsCmd);
|
|
3206
3286
|
var rawEventsStatusCmd = new Command("raw-events-status").configureHelp(helpStyle).description("Show pending raw-event backlog by source stream").option("-n, --limit <n>", "max rows to show", "25");
|
|
3207
3287
|
addDbOption(rawEventsStatusCmd);
|
|
3208
3288
|
addJsonOption(rawEventsStatusCmd);
|
|
3209
3289
|
rawEventsStatusCmd.action((opts) => {
|
|
3210
|
-
|
|
3211
|
-
|
|
3212
|
-
|
|
3213
|
-
|
|
3214
|
-
|
|
3215
|
-
|
|
3216
|
-
|
|
3217
|
-
|
|
3218
|
-
|
|
3219
|
-
|
|
3290
|
+
try {
|
|
3291
|
+
const result = getRawEventStatus(resolveDbOpt(opts), Number.parseInt(opts.limit, 10) || 25);
|
|
3292
|
+
if (opts.json) {
|
|
3293
|
+
console.log(JSON.stringify(result, null, 2));
|
|
3294
|
+
return;
|
|
3295
|
+
}
|
|
3296
|
+
p.intro("codemem db raw-events-status");
|
|
3297
|
+
p.log.info(`Totals: ${result.totals.pending.toLocaleString()} pending across ${result.totals.sessions.toLocaleString()} session(s)`);
|
|
3298
|
+
if (result.items.length === 0) {
|
|
3299
|
+
p.outro("No pending raw events");
|
|
3300
|
+
return;
|
|
3301
|
+
}
|
|
3302
|
+
for (const item of result.items) p.log.message(`${item.source}:${item.stream_id} pending=${Math.max(0, item.last_received_event_seq - item.last_flushed_event_seq)} received=${item.last_received_event_seq} flushed=${item.last_flushed_event_seq} project=${item.project ?? ""}`);
|
|
3303
|
+
p.outro("done");
|
|
3304
|
+
} catch (error) {
|
|
3305
|
+
const message = error instanceof Error ? error.message : "Failed to read raw-event status";
|
|
3306
|
+
if (opts.json) emitJsonError("raw_events_status_failed", message);
|
|
3307
|
+
else {
|
|
3308
|
+
p.log.error(message);
|
|
3309
|
+
process.exitCode = 1;
|
|
3310
|
+
}
|
|
3220
3311
|
}
|
|
3221
|
-
for (const item of result.items) p.log.message(`${item.source}:${item.stream_id} pending=${Math.max(0, item.last_received_event_seq - item.last_flushed_event_seq)} received=${item.last_received_event_seq} flushed=${item.last_flushed_event_seq} project=${item.project ?? ""}`);
|
|
3222
|
-
p.outro("done");
|
|
3223
3312
|
});
|
|
3224
3313
|
dbCommand.addCommand(rawEventsStatusCmd);
|
|
3225
3314
|
var rawEventsRetryCmd = new Command("raw-events-retry").configureHelp(helpStyle).description("Requeue failed raw-event flush batches").option("-n, --limit <n>", "max failed batches to requeue", "25");
|
|
3226
3315
|
addDbOption(rawEventsRetryCmd);
|
|
3227
3316
|
rawEventsRetryCmd.action((opts) => {
|
|
3228
|
-
|
|
3229
|
-
|
|
3230
|
-
|
|
3317
|
+
try {
|
|
3318
|
+
const result = retryRawEventFailures(resolveDbOpt(opts), Number.parseInt(opts.limit, 10) || 25);
|
|
3319
|
+
p.intro("codemem db raw-events-retry");
|
|
3320
|
+
p.outro(`Requeued ${result.retried.toLocaleString()} failed batch(es)`);
|
|
3321
|
+
} catch (error) {
|
|
3322
|
+
p.log.error(error instanceof Error ? error.message : String(error));
|
|
3323
|
+
process.exitCode = 1;
|
|
3324
|
+
}
|
|
3231
3325
|
});
|
|
3232
3326
|
dbCommand.addCommand(rawEventsRetryCmd);
|
|
3233
3327
|
var rawEventsGateCmd = new Command("raw-events-gate").configureHelp(helpStyle).description("Validate raw-event reliability thresholds (non-zero exit on failure)").option("--min-flush-success-rate <rate>", "minimum flush success rate", "0.95").option("--max-dropped-event-rate <rate>", "maximum dropped event rate", "0.05").option("--min-session-boundary-accuracy <rate>", "minimum session boundary accuracy", "0.9").option("--window-hours <hours>", "lookback window in hours", "24");
|
|
@@ -3287,6 +3381,9 @@ renameProjectCmd.action((oldName, newName, opts) => {
|
|
|
3287
3381
|
p.log.info([`Sessions: ${counts.sessions}`, `Raw event sessions: ${counts.raw_event_sessions}`].join("\n"));
|
|
3288
3382
|
if (dryRun) p.outro("Pass --apply to execute");
|
|
3289
3383
|
else p.outro("done");
|
|
3384
|
+
} catch (error) {
|
|
3385
|
+
p.log.error(error instanceof Error ? error.message : String(error));
|
|
3386
|
+
process.exitCode = 1;
|
|
3290
3387
|
} finally {
|
|
3291
3388
|
store.close();
|
|
3292
3389
|
}
|
|
@@ -3332,6 +3429,9 @@ normalizeProjectsCmd.action((opts) => {
|
|
|
3332
3429
|
}
|
|
3333
3430
|
if (dryRun) p.outro("Pass --apply to execute");
|
|
3334
3431
|
else p.outro("done");
|
|
3432
|
+
} catch (error) {
|
|
3433
|
+
p.log.error(error instanceof Error ? error.message : String(error));
|
|
3434
|
+
process.exitCode = 1;
|
|
3335
3435
|
} finally {
|
|
3336
3436
|
store.close();
|
|
3337
3437
|
}
|
|
@@ -4429,58 +4529,68 @@ function createMemoryRoleReportCommand() {
|
|
|
4429
4529
|
addDbOption(cmd);
|
|
4430
4530
|
addJsonOption(cmd);
|
|
4431
4531
|
cmd.action((opts) => {
|
|
4432
|
-
|
|
4433
|
-
|
|
4434
|
-
|
|
4435
|
-
|
|
4436
|
-
|
|
4437
|
-
|
|
4438
|
-
|
|
4439
|
-
|
|
4440
|
-
|
|
4441
|
-
|
|
4442
|
-
|
|
4443
|
-
|
|
4444
|
-
|
|
4445
|
-
|
|
4446
|
-
|
|
4447
|
-
|
|
4448
|
-
|
|
4449
|
-
|
|
4450
|
-
|
|
4451
|
-
|
|
4452
|
-
|
|
4453
|
-
|
|
4454
|
-
|
|
4455
|
-
|
|
4456
|
-
|
|
4457
|
-
|
|
4458
|
-
|
|
4459
|
-
|
|
4460
|
-
|
|
4461
|
-
|
|
4462
|
-
|
|
4463
|
-
|
|
4464
|
-
|
|
4465
|
-
|
|
4466
|
-
|
|
4467
|
-
|
|
4468
|
-
|
|
4469
|
-
|
|
4470
|
-
|
|
4471
|
-
|
|
4472
|
-
|
|
4473
|
-
|
|
4474
|
-
|
|
4475
|
-
|
|
4476
|
-
|
|
4477
|
-
|
|
4478
|
-
|
|
4479
|
-
|
|
4480
|
-
|
|
4532
|
+
try {
|
|
4533
|
+
const project = opts.allProjects === true ? null : opts.project?.trim() || process.env.CODEMEM_PROJECT?.trim() || resolveProject(process.cwd(), null);
|
|
4534
|
+
const invalidScenario = (opts.scenario ?? []).find((id) => getInjectionEvalScenarioPack(id) == null);
|
|
4535
|
+
if (invalidScenario) throw new Error(`Unknown eval scenario pack: ${invalidScenario}`);
|
|
4536
|
+
const probes = [...opts.probe ?? [], ...getInjectionEvalScenarioPrompts(opts.scenario ?? [])];
|
|
4537
|
+
const result = getMemoryRoleReport(resolveDbOpt(opts), {
|
|
4538
|
+
project,
|
|
4539
|
+
allProjects: opts.allProjects === true,
|
|
4540
|
+
includeInactive: opts.inactive === true,
|
|
4541
|
+
probes
|
|
4542
|
+
});
|
|
4543
|
+
if (opts.json) {
|
|
4544
|
+
console.log(JSON.stringify(result, null, 2));
|
|
4545
|
+
return;
|
|
4546
|
+
}
|
|
4547
|
+
p.intro("codemem memory role-report");
|
|
4548
|
+
p.log.info([
|
|
4549
|
+
`Memories: ${result.totals.memories}`,
|
|
4550
|
+
`Active: ${result.totals.active}`,
|
|
4551
|
+
`Sessions: ${result.totals.sessions}`
|
|
4552
|
+
].join("\n"));
|
|
4553
|
+
p.log.info("Counts by role:");
|
|
4554
|
+
for (const [role, count] of Object.entries(result.counts_by_role)) p.log.message(` ${role.padEnd(10)} ${String(count)}`);
|
|
4555
|
+
p.log.info("Counts by mapping:");
|
|
4556
|
+
p.log.message(` mapped ${result.counts_by_mapping.mapped}`);
|
|
4557
|
+
p.log.message(` unmapped ${result.counts_by_mapping.unmapped}`);
|
|
4558
|
+
p.log.info("Summary lineages:");
|
|
4559
|
+
p.log.message(` session_summary ${result.summary_lineages.session_summary}`);
|
|
4560
|
+
p.log.message(` legacy_metadata_summary ${result.summary_lineages.legacy_metadata_summary}`);
|
|
4561
|
+
p.log.message(` summary_mapped ${result.summary_mapping.mapped}`);
|
|
4562
|
+
p.log.message(` summary_unmapped ${result.summary_mapping.unmapped}`);
|
|
4563
|
+
p.log.info("Project quality:");
|
|
4564
|
+
for (const [bucket, count] of Object.entries(result.project_quality)) p.log.message(` ${bucket.padEnd(12)} ${String(count)}`);
|
|
4565
|
+
p.log.info("Session classes:");
|
|
4566
|
+
for (const [bucket, count] of Object.entries(result.session_class_buckets)) p.log.message(` ${bucket.padEnd(20)} ${String(count)}`);
|
|
4567
|
+
p.log.info("Summary dispositions:");
|
|
4568
|
+
for (const [bucket, count] of Object.entries(result.summary_disposition_buckets)) p.log.message(` ${bucket.padEnd(20)} ${String(count)}`);
|
|
4569
|
+
if (result.probe_results.length > 0) {
|
|
4570
|
+
p.log.info("Probe results:");
|
|
4571
|
+
for (const probe of result.probe_results) {
|
|
4572
|
+
p.log.message(` query: ${probe.query}`);
|
|
4573
|
+
if (probe.scenario_id) p.log.message(` scenario: ${probe.scenario_id} (${probe.scenario_category ?? "unknown"})${probe.scenario_title ? ` — ${probe.scenario_title}` : ""}`);
|
|
4574
|
+
p.log.message(` mode: ${probe.mode}`);
|
|
4575
|
+
p.log.message(` top roles: durable=${probe.top_role_counts.durable} recap=${probe.top_role_counts.recap} ephemeral=${probe.top_role_counts.ephemeral} general=${probe.top_role_counts.general}`);
|
|
4576
|
+
p.log.message(` top mapping: mapped=${probe.top_mapping_counts.mapped} unmapped=${probe.top_mapping_counts.unmapped}`);
|
|
4577
|
+
p.log.message(` burden: recap_share=${probe.top_burden.recap_share.toFixed(2)} unmapped_share=${probe.top_burden.unmapped_share.toFixed(2)} recap_unmapped_share=${probe.top_burden.recap_unmapped_share.toFixed(2)}`);
|
|
4578
|
+
if (probe.simulated_demoted_unmapped_recap) p.log.message(` simulated demote-unmapped-recap burden: recap_share=${probe.simulated_demoted_unmapped_recap.top_burden.recap_share.toFixed(2)} unmapped_share=${probe.simulated_demoted_unmapped_recap.top_burden.unmapped_share.toFixed(2)} recap_unmapped_share=${probe.simulated_demoted_unmapped_recap.top_burden.recap_unmapped_share.toFixed(2)}`);
|
|
4579
|
+
if (probe.simulated_demoted_unmapped_recap_and_ephemeral) p.log.message(` simulated demote-unmapped-recap+ephemeral burden: recap_share=${probe.simulated_demoted_unmapped_recap_and_ephemeral.top_burden.recap_share.toFixed(2)} unmapped_share=${probe.simulated_demoted_unmapped_recap_and_ephemeral.top_burden.unmapped_share.toFixed(2)} recap_unmapped_share=${probe.simulated_demoted_unmapped_recap_and_ephemeral.top_burden.recap_unmapped_share.toFixed(2)}`);
|
|
4580
|
+
if (probe.scenario_score) p.log.message(` scenario score: mode_match=${probe.scenario_score.mode_match ? "yes" : "no"} top1_primary=${probe.scenario_score.primary_in_top1 ? "yes" : "no"} top3_primary=${probe.scenario_score.primary_in_top3_count} top1_anti=${probe.scenario_score.anti_signal_in_top1 ? "yes" : "no"} primary=${probe.scenario_score.primary_match_count} anti=${probe.scenario_score.anti_signal_count} recap=${probe.scenario_score.recap_count} unmapped_recap=${probe.scenario_score.unmapped_recap_count} chatter=${probe.scenario_score.administrative_chatter_count} net=${probe.scenario_score.score}`);
|
|
4581
|
+
for (const item of probe.items.slice(0, 5)) p.log.message(` [${item.id}] (${item.kind}/${item.role}/${item.mapping}/${item.session_class}/${item.summary_disposition}) ${item.title} — ${item.role_reason}`);
|
|
4582
|
+
}
|
|
4583
|
+
}
|
|
4584
|
+
p.outro("done");
|
|
4585
|
+
} catch (error) {
|
|
4586
|
+
const message = error instanceof Error ? error.message : "Role report failed";
|
|
4587
|
+
if (opts.json) emitJsonError("role_report_failed", message);
|
|
4588
|
+
else {
|
|
4589
|
+
p.log.error(message);
|
|
4590
|
+
process.exitCode = 1;
|
|
4481
4591
|
}
|
|
4592
|
+
return;
|
|
4482
4593
|
}
|
|
4483
|
-
p.outro("done");
|
|
4484
4594
|
});
|
|
4485
4595
|
return cmd;
|
|
4486
4596
|
}
|
|
@@ -4488,49 +4598,59 @@ function createMemoryRoleCompareCommand() {
|
|
|
4488
4598
|
const cmd = new Command("role-compare").configureHelp(helpStyle).description("Compare inferred memory-role and probe metrics across two DB snapshots").argument("<baseline_db>", "baseline sqlite database path").argument("<candidate_db>", "candidate sqlite database path").option("--project <project>", "project identifier (defaults to git repo root)").option("--all-projects", "analyze across all projects").option("--probe <query>", "run a retrieval probe query against both snapshots", (value, prev) => [...prev, value], []).option("--scenario <id>", "run a named injection-first eval scenario pack (can be repeated)", (value, prev) => [...prev, value], []).option("--inactive", "include inactive memories");
|
|
4489
4599
|
addJsonOption(cmd);
|
|
4490
4600
|
cmd.action((baselineDb, candidateDb, opts) => {
|
|
4491
|
-
|
|
4492
|
-
|
|
4493
|
-
|
|
4494
|
-
|
|
4495
|
-
|
|
4496
|
-
|
|
4497
|
-
|
|
4498
|
-
|
|
4499
|
-
|
|
4500
|
-
|
|
4501
|
-
|
|
4502
|
-
|
|
4503
|
-
|
|
4504
|
-
|
|
4505
|
-
|
|
4506
|
-
|
|
4507
|
-
|
|
4508
|
-
|
|
4509
|
-
|
|
4510
|
-
|
|
4511
|
-
|
|
4512
|
-
|
|
4513
|
-
|
|
4514
|
-
|
|
4515
|
-
|
|
4516
|
-
|
|
4517
|
-
|
|
4518
|
-
|
|
4519
|
-
|
|
4520
|
-
|
|
4521
|
-
|
|
4522
|
-
|
|
4523
|
-
|
|
4524
|
-
|
|
4525
|
-
|
|
4526
|
-
|
|
4527
|
-
|
|
4528
|
-
|
|
4529
|
-
|
|
4530
|
-
|
|
4601
|
+
try {
|
|
4602
|
+
const project = opts.allProjects === true ? null : opts.project?.trim() || process.env.CODEMEM_PROJECT?.trim() || resolveProject(process.cwd(), null);
|
|
4603
|
+
const invalidScenario = (opts.scenario ?? []).find((id) => getInjectionEvalScenarioPack(id) == null);
|
|
4604
|
+
if (invalidScenario) throw new Error(`Unknown eval scenario pack: ${invalidScenario}`);
|
|
4605
|
+
const probes = [...opts.probe ?? [], ...getInjectionEvalScenarioPrompts(opts.scenario ?? [])];
|
|
4606
|
+
const result = compareMemoryRoleReports(baselineDb, candidateDb, {
|
|
4607
|
+
project,
|
|
4608
|
+
allProjects: opts.allProjects === true,
|
|
4609
|
+
includeInactive: opts.inactive === true,
|
|
4610
|
+
probes
|
|
4611
|
+
});
|
|
4612
|
+
if (opts.json) {
|
|
4613
|
+
console.log(JSON.stringify(result, null, 2));
|
|
4614
|
+
return;
|
|
4615
|
+
}
|
|
4616
|
+
p.intro("codemem memory role-compare");
|
|
4617
|
+
p.log.info([
|
|
4618
|
+
`Baseline sessions: ${result.baseline.totals.sessions}`,
|
|
4619
|
+
`Candidate sessions: ${result.candidate.totals.sessions}`,
|
|
4620
|
+
`Delta sessions: ${result.delta.totals.sessions}`,
|
|
4621
|
+
`Mapped delta: ${result.delta.counts_by_mapping.mapped}`,
|
|
4622
|
+
`Unmapped delta: ${result.delta.counts_by_mapping.unmapped}`,
|
|
4623
|
+
`Summary mapped delta: ${result.delta.summary_mapping.mapped}`,
|
|
4624
|
+
`Summary unmapped delta: ${result.delta.summary_mapping.unmapped}`
|
|
4625
|
+
].join("\n"));
|
|
4626
|
+
p.log.info("Role deltas:");
|
|
4627
|
+
for (const [role, count] of Object.entries(result.delta.counts_by_role)) p.log.message(` ${role.padEnd(10)} ${String(count)}`);
|
|
4628
|
+
p.log.info("Session class deltas:");
|
|
4629
|
+
for (const [bucket, count] of Object.entries(result.delta.session_class_buckets)) p.log.message(` ${bucket.padEnd(20)} ${String(count)}`);
|
|
4630
|
+
p.log.info("Summary disposition deltas:");
|
|
4631
|
+
for (const [bucket, count] of Object.entries(result.delta.summary_disposition_buckets)) p.log.message(` ${bucket.padEnd(20)} ${String(count)}`);
|
|
4632
|
+
if (result.probe_comparisons.length > 0) {
|
|
4633
|
+
p.log.info("Probe comparisons:");
|
|
4634
|
+
for (const probe of result.probe_comparisons) {
|
|
4635
|
+
p.log.message(` query: ${probe.query}`);
|
|
4636
|
+
p.log.message(` modes: baseline=${probe.baseline_mode ?? "-"} candidate=${probe.candidate_mode ?? "-"}`);
|
|
4637
|
+
p.log.message(` overlap: shared_top_keys=${probe.shared_item_keys.length} baseline_top=${probe.baseline_item_ids.slice(0, 5).join(",") || "-"} candidate_top=${probe.candidate_item_ids.slice(0, 5).join(",") || "-"}`);
|
|
4638
|
+
if (probe.delta_top_burden) p.log.message(` burden delta: recap_share=${probe.delta_top_burden.recap_share.toFixed(2)} unmapped_share=${probe.delta_top_burden.unmapped_share.toFixed(2)} recap_unmapped_share=${probe.delta_top_burden.recap_unmapped_share.toFixed(2)}`);
|
|
4639
|
+
if (probe.delta_top_mapping_counts) p.log.message(` mapping delta: mapped=${probe.delta_top_mapping_counts.mapped} unmapped=${probe.delta_top_mapping_counts.unmapped}`);
|
|
4640
|
+
if (probe.baseline_scenario_score || probe.candidate_scenario_score) p.log.message(` scenario scores: baseline=${probe.baseline_scenario_score?.score ?? "-"} candidate=${probe.candidate_scenario_score?.score ?? "-"}`);
|
|
4641
|
+
if (probe.delta_scenario_score) p.log.message(` scenario delta: mode_match=${probe.delta_scenario_score.mode_match ?? "-"} top1_primary=${probe.delta_scenario_score.primary_in_top1 ?? "-"} top3_primary=${probe.delta_scenario_score.primary_in_top3_count ?? "-"} top1_anti=${probe.delta_scenario_score.anti_signal_in_top1 ?? "-"} primary=${probe.delta_scenario_score.primary_match_count ?? "-"} anti=${probe.delta_scenario_score.anti_signal_count ?? "-"} recap=${probe.delta_scenario_score.recap_count ?? "-"} unmapped_recap=${probe.delta_scenario_score.unmapped_recap_count ?? "-"} chatter=${probe.delta_scenario_score.administrative_chatter_count ?? "-"} net=${probe.delta_scenario_score.score ?? "-"}`);
|
|
4642
|
+
}
|
|
4643
|
+
}
|
|
4644
|
+
p.outro("done");
|
|
4645
|
+
} catch (error) {
|
|
4646
|
+
const message = error instanceof Error ? error.message : "Role compare failed";
|
|
4647
|
+
if (opts.json) emitJsonError("role_compare_failed", message);
|
|
4648
|
+
else {
|
|
4649
|
+
p.log.error(message);
|
|
4650
|
+
process.exitCode = 1;
|
|
4531
4651
|
}
|
|
4652
|
+
return;
|
|
4532
4653
|
}
|
|
4533
|
-
p.outro("done");
|
|
4534
4654
|
});
|
|
4535
4655
|
return cmd;
|
|
4536
4656
|
}
|
|
@@ -4539,55 +4659,65 @@ function createMemoryExtractionReportCommand() {
|
|
|
4539
4659
|
addDbOption(cmd);
|
|
4540
4660
|
addJsonOption(cmd);
|
|
4541
4661
|
cmd.action((opts) => {
|
|
4542
|
-
|
|
4543
|
-
|
|
4544
|
-
|
|
4545
|
-
|
|
4546
|
-
|
|
4547
|
-
|
|
4548
|
-
|
|
4549
|
-
|
|
4550
|
-
|
|
4551
|
-
|
|
4552
|
-
|
|
4553
|
-
|
|
4554
|
-
|
|
4555
|
-
batchId,
|
|
4556
|
-
|
|
4557
|
-
|
|
4558
|
-
|
|
4559
|
-
|
|
4560
|
-
|
|
4561
|
-
|
|
4562
|
-
|
|
4563
|
-
|
|
4564
|
-
|
|
4662
|
+
try {
|
|
4663
|
+
const sessionIdInput = opts.sessionId?.trim() ?? "";
|
|
4664
|
+
const batchIdInput = opts.batchId?.trim() ?? "";
|
|
4665
|
+
const hasSessionId = sessionIdInput.length > 0;
|
|
4666
|
+
const hasBatchId = batchIdInput.length > 0;
|
|
4667
|
+
if (hasSessionId === hasBatchId) throw new Error("Provide exactly one of --session-id or --batch-id");
|
|
4668
|
+
const sessionId = hasSessionId ? parseStrictPositiveId(sessionIdInput) : null;
|
|
4669
|
+
if (hasSessionId && sessionId === null) throw new Error(`Invalid session ID: ${sessionIdInput || opts.sessionId}`);
|
|
4670
|
+
const batchId = hasBatchId ? parseStrictPositiveId(batchIdInput) : null;
|
|
4671
|
+
if (hasBatchId && batchId === null) throw new Error(`Invalid batch ID: ${batchIdInput || opts.batchId}`);
|
|
4672
|
+
const scenarioId = opts.scenario?.trim() ?? "";
|
|
4673
|
+
const scenario = getSessionExtractionEvalScenario(scenarioId);
|
|
4674
|
+
if (!scenario) throw new Error(`Unknown extraction eval scenario: ${scenarioId || opts.scenario}`);
|
|
4675
|
+
const result = batchId != null ? getSessionExtractionEval(resolveDbOpt(opts), {
|
|
4676
|
+
batchId,
|
|
4677
|
+
scenarioId: scenario.id,
|
|
4678
|
+
includeInactive: opts.inactive === true
|
|
4679
|
+
}) : getSessionExtractionEval(resolveDbOpt(opts), {
|
|
4680
|
+
sessionId,
|
|
4681
|
+
scenarioId: scenario.id,
|
|
4682
|
+
includeInactive: opts.inactive === true
|
|
4683
|
+
});
|
|
4684
|
+
if (opts.json) {
|
|
4685
|
+
console.log(JSON.stringify(result, null, 2));
|
|
4686
|
+
return;
|
|
4687
|
+
}
|
|
4688
|
+
p.intro("codemem memory extraction-report");
|
|
4689
|
+
p.log.info([
|
|
4690
|
+
`Scenario: ${result.scenario.id} — ${result.scenario.title}`,
|
|
4691
|
+
`Target: ${result.target.type}${result.target.batchId != null ? ` #${result.target.batchId}` : ""}`,
|
|
4692
|
+
`Session: ${result.session.id} (${result.session.project ?? "no-project"})`,
|
|
4693
|
+
`Session class: ${result.session.sessionClass}`,
|
|
4694
|
+
`Summary disposition: ${result.session.summaryDisposition}`
|
|
4695
|
+
].join("\n"));
|
|
4696
|
+
p.log.info([
|
|
4697
|
+
`Pass: ${result.pass ? "yes" : "no"}`,
|
|
4698
|
+
`Summary count: ${result.counts.summaries}`,
|
|
4699
|
+
`Observation count: ${result.counts.observations}`,
|
|
4700
|
+
`Summary thread coverage: ${result.coverage.summaryThreadCoverage}`,
|
|
4701
|
+
`Observation thread coverage: ${result.coverage.observationThreadCoverage}`,
|
|
4702
|
+
`Total thread coverage: ${result.coverage.totalThreadCoverage}`,
|
|
4703
|
+
`Duplicate observation threads: ${result.coverage.duplicateObservationThreads}`
|
|
4704
|
+
].join("\n"));
|
|
4705
|
+
if (result.failureReasons.length > 0) {
|
|
4706
|
+
p.log.warn("Failure reasons:");
|
|
4707
|
+
for (const reason of result.failureReasons) p.log.message(` - ${reason}`);
|
|
4708
|
+
}
|
|
4709
|
+
p.log.info("Thread coverage:");
|
|
4710
|
+
for (const thread of result.threads) p.log.message(` ${thread.id.padEnd(22)} summary=${thread.summaryMatch ? "yes" : "no"} observations=${thread.observationMatch ? "yes" : "no"}`);
|
|
4711
|
+
p.outro("done");
|
|
4712
|
+
} catch (error) {
|
|
4713
|
+
const message = error instanceof Error ? error.message : "Extraction report failed";
|
|
4714
|
+
if (opts.json) emitJsonError("extraction_report_failed", message);
|
|
4715
|
+
else {
|
|
4716
|
+
p.log.error(message);
|
|
4717
|
+
process.exitCode = 1;
|
|
4718
|
+
}
|
|
4565
4719
|
return;
|
|
4566
4720
|
}
|
|
4567
|
-
p.intro("codemem memory extraction-report");
|
|
4568
|
-
p.log.info([
|
|
4569
|
-
`Scenario: ${result.scenario.id} — ${result.scenario.title}`,
|
|
4570
|
-
`Target: ${result.target.type}${result.target.batchId != null ? ` #${result.target.batchId}` : ""}`,
|
|
4571
|
-
`Session: ${result.session.id} (${result.session.project ?? "no-project"})`,
|
|
4572
|
-
`Session class: ${result.session.sessionClass}`,
|
|
4573
|
-
`Summary disposition: ${result.session.summaryDisposition}`
|
|
4574
|
-
].join("\n"));
|
|
4575
|
-
p.log.info([
|
|
4576
|
-
`Pass: ${result.pass ? "yes" : "no"}`,
|
|
4577
|
-
`Summary count: ${result.counts.summaries}`,
|
|
4578
|
-
`Observation count: ${result.counts.observations}`,
|
|
4579
|
-
`Summary thread coverage: ${result.coverage.summaryThreadCoverage}`,
|
|
4580
|
-
`Observation thread coverage: ${result.coverage.observationThreadCoverage}`,
|
|
4581
|
-
`Total thread coverage: ${result.coverage.totalThreadCoverage}`,
|
|
4582
|
-
`Duplicate observation threads: ${result.coverage.duplicateObservationThreads}`
|
|
4583
|
-
].join("\n"));
|
|
4584
|
-
if (result.failureReasons.length > 0) {
|
|
4585
|
-
p.log.warn("Failure reasons:");
|
|
4586
|
-
for (const reason of result.failureReasons) p.log.message(` - ${reason}`);
|
|
4587
|
-
}
|
|
4588
|
-
p.log.info("Thread coverage:");
|
|
4589
|
-
for (const thread of result.threads) p.log.message(` ${thread.id.padEnd(22)} summary=${thread.summaryMatch ? "yes" : "no"} observations=${thread.observationMatch ? "yes" : "no"}`);
|
|
4590
|
-
p.outro("done");
|
|
4591
4721
|
});
|
|
4592
4722
|
return cmd;
|
|
4593
4723
|
}
|
|
@@ -4596,73 +4726,83 @@ function createMemoryExtractionReplayCommand() {
|
|
|
4596
4726
|
addDbOption(cmd);
|
|
4597
4727
|
addJsonOption(cmd);
|
|
4598
4728
|
cmd.action(async (opts) => {
|
|
4599
|
-
|
|
4600
|
-
|
|
4601
|
-
|
|
4602
|
-
|
|
4603
|
-
|
|
4604
|
-
|
|
4605
|
-
|
|
4606
|
-
|
|
4607
|
-
|
|
4608
|
-
|
|
4609
|
-
|
|
4610
|
-
|
|
4611
|
-
|
|
4612
|
-
|
|
4613
|
-
|
|
4614
|
-
|
|
4615
|
-
|
|
4616
|
-
|
|
4617
|
-
|
|
4618
|
-
|
|
4619
|
-
|
|
4620
|
-
|
|
4621
|
-
|
|
4622
|
-
|
|
4623
|
-
|
|
4624
|
-
|
|
4625
|
-
|
|
4626
|
-
|
|
4627
|
-
|
|
4628
|
-
|
|
4629
|
-
|
|
4630
|
-
|
|
4631
|
-
|
|
4632
|
-
|
|
4633
|
-
|
|
4634
|
-
|
|
4635
|
-
|
|
4636
|
-
|
|
4637
|
-
|
|
4638
|
-
|
|
4729
|
+
try {
|
|
4730
|
+
const batchIdInput = opts.batchId?.trim() ?? "";
|
|
4731
|
+
const batchId = parseStrictPositiveId(batchIdInput);
|
|
4732
|
+
if (batchId === null) throw new Error(`Invalid batch ID: ${batchIdInput || opts.batchId}`);
|
|
4733
|
+
const scenarioId = opts.scenario?.trim() ?? "";
|
|
4734
|
+
const scenario = getSessionExtractionEvalScenario(scenarioId);
|
|
4735
|
+
if (!scenario) throw new Error(`Unknown extraction eval scenario: ${scenarioId || opts.scenario}`);
|
|
4736
|
+
const transcriptBudgetInput = opts.transcriptBudget?.trim() ?? "";
|
|
4737
|
+
const transcriptBudget = transcriptBudgetInput.length > 0 ? parseStrictPositiveId(transcriptBudgetInput) : null;
|
|
4738
|
+
if (transcriptBudgetInput.length > 0 && transcriptBudget === null) throw new Error(`Invalid transcript budget: ${transcriptBudgetInput || opts.transcriptBudget}`);
|
|
4739
|
+
const observerTemperatureInput = opts.observerTemperature?.trim() ?? "";
|
|
4740
|
+
let observerTemperature;
|
|
4741
|
+
if (observerTemperatureInput.length > 0) {
|
|
4742
|
+
const parsed = Number(observerTemperatureInput);
|
|
4743
|
+
if (!Number.isFinite(parsed)) throw new Error(`Invalid observer temperature: ${observerTemperatureInput || opts.observerTemperature}`);
|
|
4744
|
+
observerTemperature = parsed;
|
|
4745
|
+
}
|
|
4746
|
+
const maxOutputTokensInput = opts.maxOutputTokens?.trim() ?? "";
|
|
4747
|
+
const maxOutputTokens = maxOutputTokensInput.length > 0 ? parseStrictPositiveId(maxOutputTokensInput) : null;
|
|
4748
|
+
if (maxOutputTokensInput.length > 0 && maxOutputTokens === null) throw new Error(`Invalid max output tokens: ${maxOutputTokensInput || opts.maxOutputTokens}`);
|
|
4749
|
+
const observerConfig = loadObserverConfig();
|
|
4750
|
+
const observerConfigWithOverrides = {
|
|
4751
|
+
...observerConfig,
|
|
4752
|
+
observerTemperature: observerTemperature ?? observerConfig.observerTemperature,
|
|
4753
|
+
observerOpenAIUseResponses: opts.openaiResponses === true,
|
|
4754
|
+
observerReasoningEffort: opts.reasoningEffort?.trim() || null,
|
|
4755
|
+
observerReasoningSummary: opts.reasoningSummary?.trim() || null,
|
|
4756
|
+
observerMaxOutputTokens: maxOutputTokens ?? observerConfig.observerMaxTokens
|
|
4757
|
+
};
|
|
4758
|
+
const observer = new ObserverClient(observerConfigWithOverrides);
|
|
4759
|
+
const result = opts.observerTierRouting === true ? await replayBatchExtractionWithTierRouting(resolveDbOpt(opts), observerConfigWithOverrides, {
|
|
4760
|
+
batchId,
|
|
4761
|
+
scenarioId: scenario.id,
|
|
4762
|
+
transcriptBudget: transcriptBudget ?? void 0
|
|
4763
|
+
}) : await replayBatchExtraction(resolveDbOpt(opts), observer, {
|
|
4764
|
+
batchId,
|
|
4765
|
+
scenarioId: scenario.id,
|
|
4766
|
+
transcriptBudget: transcriptBudget ?? void 0
|
|
4767
|
+
});
|
|
4768
|
+
if (opts.json) {
|
|
4769
|
+
console.log(JSON.stringify(result, null, 2));
|
|
4770
|
+
return;
|
|
4771
|
+
}
|
|
4772
|
+
p.intro("codemem memory extraction-replay");
|
|
4773
|
+
p.log.info([
|
|
4774
|
+
`Scenario: ${result.scenario.id} — ${result.scenario.title}`,
|
|
4775
|
+
`Batch: ${result.target.batchId}`,
|
|
4776
|
+
`Session: ${result.target.sessionId}`,
|
|
4777
|
+
`Observer: ${result.observer.provider}/${result.observer.model}`,
|
|
4778
|
+
`Tier: ${result.observer.tier ?? "manual"}`,
|
|
4779
|
+
`OpenAI Responses: ${result.observer.openaiUseResponses ? "yes" : "no"}`,
|
|
4780
|
+
`Reasoning effort: ${result.observer.reasoningEffort ?? "none"}`,
|
|
4781
|
+
`Classification: ${result.classification.status}`,
|
|
4782
|
+
`Pass: ${result.evaluation.pass ? "yes" : "no"}`
|
|
4783
|
+
].join("\n"));
|
|
4784
|
+
if (result.classification.reason) p.log.message(`Classification reason: ${result.classification.reason}`);
|
|
4785
|
+
if (result.evaluation.failureReasons.length > 0) {
|
|
4786
|
+
p.log.warn("Failure reasons:");
|
|
4787
|
+
for (const reason of result.evaluation.failureReasons) p.log.message(` - ${reason}`);
|
|
4788
|
+
}
|
|
4789
|
+
p.log.info([
|
|
4790
|
+
`Fresh summaries: ${result.evaluation.counts.summaries}`,
|
|
4791
|
+
`Fresh observations: ${result.evaluation.counts.observations}`,
|
|
4792
|
+
`Summary thread coverage: ${result.evaluation.coverage.summaryThreadCoverage}`,
|
|
4793
|
+
`Observation thread coverage: ${result.evaluation.coverage.observationThreadCoverage}`,
|
|
4794
|
+
`Total thread coverage: ${result.evaluation.coverage.totalThreadCoverage}`
|
|
4795
|
+
].join("\n"));
|
|
4796
|
+
p.outro("done");
|
|
4797
|
+
} catch (error) {
|
|
4798
|
+
const message = error instanceof Error ? error.message : "Extraction replay failed";
|
|
4799
|
+
if (opts.json) emitJsonError("extraction_replay_failed", message);
|
|
4800
|
+
else {
|
|
4801
|
+
p.log.error(message);
|
|
4802
|
+
process.exitCode = 1;
|
|
4803
|
+
}
|
|
4639
4804
|
return;
|
|
4640
4805
|
}
|
|
4641
|
-
p.intro("codemem memory extraction-replay");
|
|
4642
|
-
p.log.info([
|
|
4643
|
-
`Scenario: ${result.scenario.id} — ${result.scenario.title}`,
|
|
4644
|
-
`Batch: ${result.target.batchId}`,
|
|
4645
|
-
`Session: ${result.target.sessionId}`,
|
|
4646
|
-
`Observer: ${result.observer.provider}/${result.observer.model}`,
|
|
4647
|
-
`Tier: ${result.observer.tier ?? "manual"}`,
|
|
4648
|
-
`OpenAI Responses: ${result.observer.openaiUseResponses ? "yes" : "no"}`,
|
|
4649
|
-
`Reasoning effort: ${result.observer.reasoningEffort ?? "none"}`,
|
|
4650
|
-
`Classification: ${result.classification.status}`,
|
|
4651
|
-
`Pass: ${result.evaluation.pass ? "yes" : "no"}`
|
|
4652
|
-
].join("\n"));
|
|
4653
|
-
if (result.classification.reason) p.log.message(`Classification reason: ${result.classification.reason}`);
|
|
4654
|
-
if (result.evaluation.failureReasons.length > 0) {
|
|
4655
|
-
p.log.warn("Failure reasons:");
|
|
4656
|
-
for (const reason of result.evaluation.failureReasons) p.log.message(` - ${reason}`);
|
|
4657
|
-
}
|
|
4658
|
-
p.log.info([
|
|
4659
|
-
`Fresh summaries: ${result.evaluation.counts.summaries}`,
|
|
4660
|
-
`Fresh observations: ${result.evaluation.counts.observations}`,
|
|
4661
|
-
`Summary thread coverage: ${result.evaluation.coverage.summaryThreadCoverage}`,
|
|
4662
|
-
`Observation thread coverage: ${result.evaluation.coverage.observationThreadCoverage}`,
|
|
4663
|
-
`Total thread coverage: ${result.evaluation.coverage.totalThreadCoverage}`
|
|
4664
|
-
].join("\n"));
|
|
4665
|
-
p.outro("done");
|
|
4666
4806
|
});
|
|
4667
4807
|
return cmd;
|
|
4668
4808
|
}
|
|
@@ -4671,140 +4811,150 @@ function createMemoryExtractionBenchmarkCommand() {
|
|
|
4671
4811
|
addDbOption(cmd);
|
|
4672
4812
|
addJsonOption(cmd);
|
|
4673
4813
|
cmd.action(async (opts) => {
|
|
4674
|
-
|
|
4675
|
-
|
|
4676
|
-
|
|
4677
|
-
|
|
4678
|
-
|
|
4679
|
-
|
|
4680
|
-
|
|
4681
|
-
|
|
4682
|
-
|
|
4683
|
-
|
|
4684
|
-
|
|
4685
|
-
|
|
4686
|
-
|
|
4687
|
-
|
|
4688
|
-
|
|
4689
|
-
|
|
4690
|
-
|
|
4691
|
-
|
|
4692
|
-
|
|
4693
|
-
|
|
4694
|
-
|
|
4695
|
-
|
|
4696
|
-
|
|
4697
|
-
|
|
4698
|
-
|
|
4699
|
-
|
|
4700
|
-
|
|
4701
|
-
|
|
4702
|
-
|
|
4703
|
-
|
|
4704
|
-
const
|
|
4705
|
-
|
|
4706
|
-
|
|
4707
|
-
|
|
4708
|
-
|
|
4709
|
-
|
|
4710
|
-
|
|
4711
|
-
|
|
4712
|
-
|
|
4713
|
-
|
|
4714
|
-
|
|
4715
|
-
|
|
4716
|
-
|
|
4717
|
-
|
|
4718
|
-
|
|
4719
|
-
|
|
4720
|
-
|
|
4721
|
-
|
|
4722
|
-
|
|
4723
|
-
|
|
4724
|
-
|
|
4725
|
-
|
|
4726
|
-
|
|
4814
|
+
try {
|
|
4815
|
+
const benchmarkId = opts.benchmark?.trim() ?? "";
|
|
4816
|
+
const benchmark = getExtractionBenchmarkProfile(benchmarkId);
|
|
4817
|
+
if (!benchmark) throw new Error(`Unknown extraction benchmark: ${benchmarkId || opts.benchmark}`);
|
|
4818
|
+
const transcriptBudgetInput = opts.transcriptBudget?.trim() ?? "";
|
|
4819
|
+
const transcriptBudget = transcriptBudgetInput.length > 0 ? parseStrictPositiveId(transcriptBudgetInput) : null;
|
|
4820
|
+
if (transcriptBudgetInput.length > 0 && transcriptBudget === null) throw new Error(`Invalid transcript budget: ${transcriptBudgetInput || opts.transcriptBudget}`);
|
|
4821
|
+
const observerTemperatureInput = opts.observerTemperature?.trim() ?? "";
|
|
4822
|
+
let observerTemperature;
|
|
4823
|
+
if (observerTemperatureInput.length > 0) {
|
|
4824
|
+
const parsed = Number(observerTemperatureInput);
|
|
4825
|
+
if (!Number.isFinite(parsed)) throw new Error(`Invalid observer temperature: ${observerTemperatureInput || opts.observerTemperature}`);
|
|
4826
|
+
observerTemperature = parsed;
|
|
4827
|
+
}
|
|
4828
|
+
const maxOutputTokensInput = opts.maxOutputTokens?.trim() ?? "";
|
|
4829
|
+
const maxOutputTokens = maxOutputTokensInput.length > 0 ? parseStrictPositiveId(maxOutputTokensInput) : null;
|
|
4830
|
+
if (maxOutputTokensInput.length > 0 && maxOutputTokens === null) throw new Error(`Invalid max output tokens: ${maxOutputTokensInput || opts.maxOutputTokens}`);
|
|
4831
|
+
const observerConfig = loadObserverConfig();
|
|
4832
|
+
const observerConfigWithOverrides = {
|
|
4833
|
+
...observerConfig,
|
|
4834
|
+
observerProvider: opts.observerProvider?.trim() || observerConfig.observerProvider,
|
|
4835
|
+
observerModel: opts.observerModel?.trim() || observerConfig.observerModel,
|
|
4836
|
+
observerTemperature: observerTemperature ?? observerConfig.observerTemperature,
|
|
4837
|
+
observerOpenAIUseResponses: opts.openaiResponses === true,
|
|
4838
|
+
observerReasoningEffort: opts.reasoningEffort?.trim() || null,
|
|
4839
|
+
observerReasoningSummary: opts.reasoningSummary?.trim() || null,
|
|
4840
|
+
observerMaxOutputTokens: maxOutputTokens ?? observerConfig.observerMaxTokens
|
|
4841
|
+
};
|
|
4842
|
+
const observer = new ObserverClient(observerConfigWithOverrides);
|
|
4843
|
+
const runs = [];
|
|
4844
|
+
for (const batch of benchmark.batches) {
|
|
4845
|
+
const scenarioId = batch.scenarioId ?? benchmark.scenarioId;
|
|
4846
|
+
const result = opts.observerTierRouting === true ? await replayBatchExtractionWithTierRouting(resolveDbOpt(opts), observerConfigWithOverrides, {
|
|
4847
|
+
batchId: batch.batchId,
|
|
4848
|
+
scenarioId,
|
|
4849
|
+
transcriptBudget: transcriptBudget ?? void 0
|
|
4850
|
+
}) : await replayBatchExtraction(resolveDbOpt(opts), observer, {
|
|
4851
|
+
batchId: batch.batchId,
|
|
4852
|
+
scenarioId,
|
|
4853
|
+
transcriptBudget: transcriptBudget ?? void 0
|
|
4854
|
+
});
|
|
4855
|
+
runs.push({
|
|
4856
|
+
batchId: batch.batchId,
|
|
4857
|
+
sessionId: batch.sessionId,
|
|
4858
|
+
label: batch.label,
|
|
4859
|
+
purpose: batch.purpose,
|
|
4860
|
+
complexity: batch.complexity,
|
|
4861
|
+
scenarioId,
|
|
4862
|
+
expectedTier: batch.expectedTier ?? null,
|
|
4863
|
+
analysis: {
|
|
4864
|
+
eventSpan: result.analysis.eventSpan,
|
|
4865
|
+
promptCount: result.analysis.promptCount,
|
|
4866
|
+
toolCount: result.analysis.toolCount,
|
|
4867
|
+
transcriptLength: result.analysis.transcriptLength
|
|
4868
|
+
},
|
|
4869
|
+
status: result.classification.status,
|
|
4870
|
+
reason: result.classification.reason,
|
|
4871
|
+
tier: result.observer.tier ?? "manual",
|
|
4872
|
+
provider: result.observer.provider,
|
|
4873
|
+
model: result.observer.model,
|
|
4874
|
+
openaiUseResponses: result.observer.openaiUseResponses,
|
|
4875
|
+
reasoningEffort: result.observer.reasoningEffort,
|
|
4876
|
+
reasoningSummary: result.observer.reasoningSummary,
|
|
4877
|
+
maxOutputTokens: result.observer.maxOutputTokens,
|
|
4878
|
+
temperature: result.observer.temperature,
|
|
4879
|
+
summaries: result.evaluation.counts.summaries,
|
|
4880
|
+
observations: result.evaluation.counts.observations,
|
|
4881
|
+
repairApplied: result.observer.repairApplied
|
|
4882
|
+
});
|
|
4883
|
+
}
|
|
4884
|
+
const summary = {
|
|
4885
|
+
total: runs.length,
|
|
4886
|
+
shapeQualityTotal: runs.filter((run) => run.purpose === "shape_quality").length,
|
|
4887
|
+
shapeQualityPasses: runs.filter((run) => run.purpose === "shape_quality" && run.status === "pass").length,
|
|
4888
|
+
shapeQualityFails: runs.filter((run) => run.purpose === "shape_quality" && run.status === "shape_fail").length,
|
|
4889
|
+
expectedTierTotal: runs.filter((run) => run.expectedTier != null).length,
|
|
4890
|
+
expectedTierMatches: runs.filter((run) => run.expectedTier != null && run.expectedTier === run.tier).length,
|
|
4891
|
+
robustnessNoOutput: runs.filter((run) => run.status === "observer_no_output").length
|
|
4892
|
+
};
|
|
4893
|
+
const uniqueObserverKeys = Array.from(new Set(runs.map((run) => `${run.provider}::${run.model}::${run.openaiUseResponses ? "responses" : "chat"}`)));
|
|
4894
|
+
const observerSummary = opts.observerTierRouting === true ? {
|
|
4895
|
+
provider: uniqueObserverKeys.length === 1 ? runs[0]?.provider ?? observer.provider : "mixed",
|
|
4896
|
+
model: uniqueObserverKeys.length === 1 ? runs[0]?.model ?? observer.model : "mixed",
|
|
4897
|
+
tierRouting: true,
|
|
4898
|
+
openaiUseResponses: uniqueObserverKeys.length === 1 ? runs[0]?.openaiUseResponses ?? observer.openaiUseResponses : null,
|
|
4899
|
+
reasoningEffort: uniqueObserverKeys.length === 1 ? runs[0]?.reasoningEffort ?? observer.reasoningEffort : "mixed",
|
|
4900
|
+
reasoningSummary: uniqueObserverKeys.length === 1 ? runs[0]?.reasoningSummary ?? observer.reasoningSummary : "mixed",
|
|
4901
|
+
maxOutputTokens: uniqueObserverKeys.length === 1 ? runs[0]?.maxOutputTokens ?? observer.maxOutputTokens : null,
|
|
4902
|
+
temperature: uniqueObserverKeys.length === 1 ? runs[0]?.temperature ?? observer.temperature : null,
|
|
4903
|
+
transcriptBudget: transcriptBudget ?? null,
|
|
4904
|
+
selectedObservers: uniqueObserverKeys
|
|
4905
|
+
} : {
|
|
4906
|
+
provider: observer.provider,
|
|
4907
|
+
model: observer.model,
|
|
4908
|
+
tierRouting: false,
|
|
4909
|
+
openaiUseResponses: observer.openaiUseResponses,
|
|
4910
|
+
reasoningEffort: observer.reasoningEffort,
|
|
4911
|
+
reasoningSummary: observer.reasoningSummary,
|
|
4912
|
+
maxOutputTokens: observer.maxOutputTokens,
|
|
4913
|
+
temperature: observer.temperature,
|
|
4914
|
+
transcriptBudget: transcriptBudget ?? null,
|
|
4915
|
+
selectedObservers: uniqueObserverKeys
|
|
4916
|
+
};
|
|
4917
|
+
const output = {
|
|
4918
|
+
benchmark: {
|
|
4919
|
+
id: benchmark.id,
|
|
4920
|
+
title: benchmark.title,
|
|
4921
|
+
scenarioId: benchmark.scenarioId
|
|
4727
4922
|
},
|
|
4728
|
-
|
|
4729
|
-
|
|
4730
|
-
|
|
4731
|
-
|
|
4732
|
-
|
|
4733
|
-
|
|
4734
|
-
|
|
4735
|
-
|
|
4736
|
-
|
|
4737
|
-
|
|
4738
|
-
|
|
4739
|
-
|
|
4740
|
-
|
|
4741
|
-
|
|
4742
|
-
|
|
4743
|
-
|
|
4744
|
-
|
|
4745
|
-
|
|
4746
|
-
|
|
4747
|
-
|
|
4748
|
-
|
|
4749
|
-
|
|
4750
|
-
|
|
4751
|
-
|
|
4752
|
-
|
|
4753
|
-
|
|
4754
|
-
|
|
4755
|
-
|
|
4756
|
-
|
|
4757
|
-
|
|
4758
|
-
|
|
4759
|
-
|
|
4760
|
-
|
|
4761
|
-
temperature: uniqueObserverKeys.length === 1 ? runs[0]?.temperature ?? observer.temperature : null,
|
|
4762
|
-
transcriptBudget: transcriptBudget ?? null,
|
|
4763
|
-
selectedObservers: uniqueObserverKeys
|
|
4764
|
-
} : {
|
|
4765
|
-
provider: observer.provider,
|
|
4766
|
-
model: observer.model,
|
|
4767
|
-
tierRouting: false,
|
|
4768
|
-
openaiUseResponses: observer.openaiUseResponses,
|
|
4769
|
-
reasoningEffort: observer.reasoningEffort,
|
|
4770
|
-
reasoningSummary: observer.reasoningSummary,
|
|
4771
|
-
maxOutputTokens: observer.maxOutputTokens,
|
|
4772
|
-
temperature: observer.temperature,
|
|
4773
|
-
transcriptBudget: transcriptBudget ?? null,
|
|
4774
|
-
selectedObservers: uniqueObserverKeys
|
|
4775
|
-
};
|
|
4776
|
-
const output = {
|
|
4777
|
-
benchmark: {
|
|
4778
|
-
id: benchmark.id,
|
|
4779
|
-
title: benchmark.title,
|
|
4780
|
-
scenarioId: benchmark.scenarioId
|
|
4781
|
-
},
|
|
4782
|
-
observer: observerSummary,
|
|
4783
|
-
summary,
|
|
4784
|
-
runs
|
|
4785
|
-
};
|
|
4786
|
-
if (opts.json) {
|
|
4787
|
-
console.log(JSON.stringify(output, null, 2));
|
|
4923
|
+
observer: observerSummary,
|
|
4924
|
+
summary,
|
|
4925
|
+
runs
|
|
4926
|
+
};
|
|
4927
|
+
if (opts.json) {
|
|
4928
|
+
console.log(JSON.stringify(output, null, 2));
|
|
4929
|
+
return;
|
|
4930
|
+
}
|
|
4931
|
+
p.intro("codemem memory extraction-benchmark");
|
|
4932
|
+
p.log.info([
|
|
4933
|
+
`Benchmark: ${benchmark.id} — ${benchmark.title}`,
|
|
4934
|
+
`Observer: ${observerSummary.provider}/${observerSummary.model}`,
|
|
4935
|
+
`Tier routing: ${opts.observerTierRouting === true ? "yes" : "no"}`,
|
|
4936
|
+
`OpenAI Responses: ${observerSummary.openaiUseResponses === null ? "mixed" : observerSummary.openaiUseResponses ? "yes" : "no"}`,
|
|
4937
|
+
`Reasoning effort: ${observerSummary.reasoningEffort ?? "none"}`,
|
|
4938
|
+
`Reasoning summary: ${observerSummary.reasoningSummary ?? "none"}`,
|
|
4939
|
+
`Max output tokens: ${observerSummary.maxOutputTokens ?? "mixed"}`,
|
|
4940
|
+
`Temperature: ${observerSummary.temperature ?? "mixed"}`,
|
|
4941
|
+
`Transcript budget override: ${transcriptBudget ?? "default"}`,
|
|
4942
|
+
`Shape-quality passes: ${summary.shapeQualityPasses}/${summary.shapeQualityTotal}`,
|
|
4943
|
+
`Shape-quality fails: ${summary.shapeQualityFails}`,
|
|
4944
|
+
`Expected-tier matches: ${summary.expectedTierMatches}/${summary.expectedTierTotal}`,
|
|
4945
|
+
`Observer no-output cases: ${summary.robustnessNoOutput}`
|
|
4946
|
+
].join("\n"));
|
|
4947
|
+
for (const run of runs) p.log.message(` [${run.batchId}] ${run.status.padEnd(18)} ${run.complexity.padEnd(10)} tier=${run.tier.padEnd(6)} expected=${(run.expectedTier ?? "n/a").padEnd(6)} span=${String(run.analysis.eventSpan).padEnd(3)} prompts=${run.analysis.promptCount} tools=${String(run.analysis.toolCount).padEnd(2)} transcript=${run.analysis.transcriptLength} ${run.provider}/${run.model}${run.openaiUseResponses ? " [responses]" : ""} summaries=${run.summaries} observations=${run.observations} repair=${run.repairApplied ? "yes" : "no"} — ${run.label}`);
|
|
4948
|
+
p.outro("done");
|
|
4949
|
+
} catch (error) {
|
|
4950
|
+
const message = error instanceof Error ? error.message : "Extraction benchmark failed";
|
|
4951
|
+
if (opts.json) emitJsonError("extraction_benchmark_failed", message);
|
|
4952
|
+
else {
|
|
4953
|
+
p.log.error(message);
|
|
4954
|
+
process.exitCode = 1;
|
|
4955
|
+
}
|
|
4788
4956
|
return;
|
|
4789
4957
|
}
|
|
4790
|
-
p.intro("codemem memory extraction-benchmark");
|
|
4791
|
-
p.log.info([
|
|
4792
|
-
`Benchmark: ${benchmark.id} — ${benchmark.title}`,
|
|
4793
|
-
`Observer: ${observerSummary.provider}/${observerSummary.model}`,
|
|
4794
|
-
`Tier routing: ${opts.observerTierRouting === true ? "yes" : "no"}`,
|
|
4795
|
-
`OpenAI Responses: ${observerSummary.openaiUseResponses === null ? "mixed" : observerSummary.openaiUseResponses ? "yes" : "no"}`,
|
|
4796
|
-
`Reasoning effort: ${observerSummary.reasoningEffort ?? "none"}`,
|
|
4797
|
-
`Reasoning summary: ${observerSummary.reasoningSummary ?? "none"}`,
|
|
4798
|
-
`Max output tokens: ${observerSummary.maxOutputTokens ?? "mixed"}`,
|
|
4799
|
-
`Temperature: ${observerSummary.temperature ?? "mixed"}`,
|
|
4800
|
-
`Transcript budget override: ${transcriptBudget ?? "default"}`,
|
|
4801
|
-
`Shape-quality passes: ${summary.shapeQualityPasses}/${summary.shapeQualityTotal}`,
|
|
4802
|
-
`Shape-quality fails: ${summary.shapeQualityFails}`,
|
|
4803
|
-
`Expected-tier matches: ${summary.expectedTierMatches}/${summary.expectedTierTotal}`,
|
|
4804
|
-
`Observer no-output cases: ${summary.robustnessNoOutput}`
|
|
4805
|
-
].join("\n"));
|
|
4806
|
-
for (const run of runs) p.log.message(` [${run.batchId}] ${run.status.padEnd(18)} ${run.complexity.padEnd(10)} tier=${run.tier.padEnd(6)} expected=${(run.expectedTier ?? "n/a").padEnd(6)} span=${String(run.analysis.eventSpan).padEnd(3)} prompts=${run.analysis.promptCount} tools=${String(run.analysis.toolCount).padEnd(2)} transcript=${run.analysis.transcriptLength} ${run.provider}/${run.model}${run.openaiUseResponses ? " [responses]" : ""} summaries=${run.summaries} observations=${run.observations} repair=${run.repairApplied ? "yes" : "no"} — ${run.label}`);
|
|
4807
|
-
p.outro("done");
|
|
4808
4958
|
});
|
|
4809
4959
|
return cmd;
|
|
4810
4960
|
}
|
|
@@ -5023,6 +5173,14 @@ cmd.action((opts) => {
|
|
|
5023
5173
|
const items = store.recent(limit, filters);
|
|
5024
5174
|
if (opts.json) console.log(JSON.stringify(items));
|
|
5025
5175
|
else for (const item of items) console.log(`#${item.id} [${item.kind}] ${item.title}`);
|
|
5176
|
+
} catch (error) {
|
|
5177
|
+
const message = error instanceof Error ? error.message : "Recent lookup failed";
|
|
5178
|
+
if (opts.json) emitJsonError("recent_failed", message);
|
|
5179
|
+
else {
|
|
5180
|
+
console.error(message);
|
|
5181
|
+
process.exitCode = 1;
|
|
5182
|
+
}
|
|
5183
|
+
return;
|
|
5026
5184
|
} finally {
|
|
5027
5185
|
store.close();
|
|
5028
5186
|
}
|
|
@@ -5064,6 +5222,14 @@ var searchCommand = searchCmd.action((query, opts) => {
|
|
|
5064
5222
|
].join("\n"));
|
|
5065
5223
|
}
|
|
5066
5224
|
p.outro("done");
|
|
5225
|
+
} catch (error) {
|
|
5226
|
+
const message = error instanceof Error ? error.message : "Search failed";
|
|
5227
|
+
if (opts.json) emitJsonError("search_failed", message);
|
|
5228
|
+
else {
|
|
5229
|
+
p.log.error(message);
|
|
5230
|
+
process.exitCode = 1;
|
|
5231
|
+
}
|
|
5232
|
+
return;
|
|
5067
5233
|
} finally {
|
|
5068
5234
|
store.close();
|
|
5069
5235
|
}
|
|
@@ -6260,6 +6426,14 @@ var statsCommand = statsCmd.action((opts) => {
|
|
|
6260
6426
|
p.log.step(`Usage\n${lines.join("\n")}`);
|
|
6261
6427
|
}
|
|
6262
6428
|
p.outro("done");
|
|
6429
|
+
} catch (error) {
|
|
6430
|
+
const message = error instanceof Error ? error.message : "Stats failed";
|
|
6431
|
+
if (opts.json) emitJsonError("stats_failed", message);
|
|
6432
|
+
else {
|
|
6433
|
+
p.log.error(message);
|
|
6434
|
+
process.exitCode = 1;
|
|
6435
|
+
}
|
|
6436
|
+
return;
|
|
6263
6437
|
} finally {
|
|
6264
6438
|
store.close();
|
|
6265
6439
|
}
|
|
@@ -6431,6 +6605,14 @@ attemptsCmd.action((opts) => {
|
|
|
6431
6605
|
return;
|
|
6432
6606
|
}
|
|
6433
6607
|
for (const row of rows) console.log(formatSyncAttempt(row));
|
|
6608
|
+
} catch (error) {
|
|
6609
|
+
const message = error instanceof Error ? error.message : "Failed to read sync attempts";
|
|
6610
|
+
if (opts.json) emitJsonError("invalid_input", message);
|
|
6611
|
+
else {
|
|
6612
|
+
p.log.error(message);
|
|
6613
|
+
process.exitCode = 1;
|
|
6614
|
+
}
|
|
6615
|
+
return;
|
|
6434
6616
|
} finally {
|
|
6435
6617
|
store.close();
|
|
6436
6618
|
}
|
|
@@ -6926,6 +7108,14 @@ enableCmd.action((opts) => {
|
|
|
6926
7108
|
`Interval: ${config.sync_interval_s ?? 120}s`
|
|
6927
7109
|
].join("\n"));
|
|
6928
7110
|
p.outro("Sync enabled — restart `codemem serve` to activate");
|
|
7111
|
+
} catch (error) {
|
|
7112
|
+
const message = error instanceof Error ? error.message : "Failed to enable sync";
|
|
7113
|
+
if (opts.json) emitJsonError("invalid_input", message);
|
|
7114
|
+
else {
|
|
7115
|
+
p.log.error(message);
|
|
7116
|
+
process.exitCode = 1;
|
|
7117
|
+
}
|
|
7118
|
+
return;
|
|
6929
7119
|
} finally {
|
|
6930
7120
|
store.close();
|
|
6931
7121
|
}
|
|
@@ -7265,22 +7455,25 @@ if (hasRootFlag("--cleanup-completion")) {
|
|
|
7265
7455
|
process.exit(0);
|
|
7266
7456
|
}
|
|
7267
7457
|
{
|
|
7458
|
+
const forwardedTail = (subcommand) => {
|
|
7459
|
+
const argv = process.argv.slice(2);
|
|
7460
|
+
const groupIdx = argv.indexOf("memory");
|
|
7461
|
+
const searchFrom = groupIdx >= 0 ? groupIdx + 1 : 0;
|
|
7462
|
+
const subIdx = argv.indexOf(subcommand, searchFrom);
|
|
7463
|
+
return subIdx >= 0 ? argv.slice(subIdx + 1) : [];
|
|
7464
|
+
};
|
|
7268
7465
|
const memExport = new Command("export").description("Export memories to a JSON file for sharing or backup").argument("<output>", "output file path (use '-' for stdout)").option("--db <path>", "database path").option("--db-path <path>", "database path").option("--project <project>", "filter by project").option("--all-projects", "export all projects").option("--include-inactive", "include deactivated memories").option("--since <iso>", "only export memories created after this ISO timestamp").configureHelp(helpStyle).allowUnknownOption(true).allowExcessArguments(true).action(async () => {
|
|
7269
|
-
const idx = process.argv.indexOf("export");
|
|
7270
|
-
const tail = idx >= 0 ? process.argv.slice(idx + 1) : [];
|
|
7271
7466
|
await exportMemoriesCommand.parseAsync([
|
|
7272
7467
|
"node",
|
|
7273
7468
|
"export-memories",
|
|
7274
|
-
...
|
|
7469
|
+
...forwardedTail("export")
|
|
7275
7470
|
]);
|
|
7276
7471
|
});
|
|
7277
7472
|
const memImport = new Command("import").description("Import memories from an exported JSON file").argument("<inputFile>", "input JSON file (use '-' for stdin)").option("--db <path>", "database path").option("--db-path <path>", "database path").option("--remap-project <path>", "remap all projects to this path on import").option("--dry-run", "preview import without writing").configureHelp(helpStyle).allowUnknownOption(true).allowExcessArguments(true).action(async () => {
|
|
7278
|
-
const idx = process.argv.indexOf("import");
|
|
7279
|
-
const tail = idx >= 0 ? process.argv.slice(idx + 1) : [];
|
|
7280
7473
|
await importMemoriesCommand.parseAsync([
|
|
7281
7474
|
"node",
|
|
7282
7475
|
"import-memories",
|
|
7283
|
-
...
|
|
7476
|
+
...forwardedTail("import")
|
|
7284
7477
|
]);
|
|
7285
7478
|
});
|
|
7286
7479
|
memoryCommand.addCommand(memExport);
|