claude-memory-layer 1.0.31 → 1.0.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -2
- package/dist/cli/index.js +1 -1
- package/package.json +11 -2
- package/scripts/postinstall-embedding-backend.cjs +16 -12
- package/AGENTS.md +0 -71
- package/CLAUDE.md +0 -30
- package/HANDOFF.md +0 -92
- package/Memo.txt +0 -558
- package/benchmarks/replay/anonymized-real-sessions.json +0 -48
- package/config/kpi-thresholds.json +0 -7
- package/context.md +0 -636
- package/docs/ARCHITECTURE_COMPARISON_AND_RECOMMENDATIONS.md +0 -627
- package/docs/HERMES_MEMORY_INGESTION_ANALYSIS.md +0 -440
- package/docs/MCP_MEMORY_SERVICE_COMPARATIVE_REVIEW.md +0 -271
- package/docs/MEMORY_USEFULNESS_AUDIT.md +0 -371
- package/docs/MEMORY_USEFULNESS_AUDIT_RAW.json +0 -80
- package/docs/MEMSEARCH_PROJECT_STRUCTURE_ANALYSIS.md +0 -333
- package/docs/MEMU_ADOPTION.md +0 -40
- package/docs/OPERATIONS.md +0 -18
- package/docs/PRODUCT_VALIDATION_MATRIX.md +0 -82
- package/docs/PROJECT_STRUCTURE_ANALYSIS.md +0 -421
- package/docs/REFACTORING_MILESTONES_AND_ISSUES.md +0 -501
- package/docs/REFACTORING_PLAN_THIN_CORE.md +0 -414
- package/docs/REFERENCE_PROJECT_ANALYSES.md +0 -25
- package/docs/SUPERLOCALMEMORY_PROJECT_STRUCTURE_ANALYSIS.md +0 -452
- package/docs/TARGET_ARCHITECTURE_AND_FOLDER_STRUCTURE.md +0 -446
- package/docs/architecture/comparison-index.md +0 -47
- package/docs/reports/codex-real-data-validation-20260505T040447Z.md +0 -46
- package/plan.md +0 -1642
- package/scripts/build.ts +0 -159
- package/scripts/bump-patch-version.sh +0 -18
- package/scripts/delete-unknown-projects.js +0 -154
- package/scripts/fix-sync-gap.js +0 -32
- package/scripts/generate-session-qrels.ts +0 -126
- package/scripts/heartbeat-memory-orchestrator.sh +0 -28
- package/scripts/replay-retrieval-benchmark.ts +0 -69
- package/scripts/report-sync-gap.js +0 -26
- package/scripts/review-queue-auto-resolve.js +0 -21
- package/scripts/sync-gap-auto-heal.sh +0 -17
- package/spec.md +0 -624
- package/specs/20260207-dashboard-upgrade/context.md +0 -38
- package/specs/20260207-dashboard-upgrade/spec.md +0 -96
- package/specs/citations-system/context.md +0 -243
- package/specs/citations-system/plan.md +0 -495
- package/specs/citations-system/spec.md +0 -371
- package/specs/endless-mode/context.md +0 -305
- package/specs/endless-mode/plan.md +0 -620
- package/specs/endless-mode/spec.md +0 -455
- package/specs/entity-edge-model/context.md +0 -401
- package/specs/entity-edge-model/plan.md +0 -459
- package/specs/entity-edge-model/spec.md +0 -391
- package/specs/evidence-aligner-v2/context.md +0 -401
- package/specs/evidence-aligner-v2/plan.md +0 -303
- package/specs/evidence-aligner-v2/spec.md +0 -312
- package/specs/mcp-desktop-integration/context.md +0 -278
- package/specs/mcp-desktop-integration/plan.md +0 -550
- package/specs/mcp-desktop-integration/spec.md +0 -494
- package/specs/memory-utilization-improvements/context.md +0 -145
- package/specs/memory-utilization-improvements/plan.md +0 -361
- package/specs/memory-utilization-improvements/spec.md +0 -361
- package/specs/post-tool-use-hook/context.md +0 -319
- package/specs/post-tool-use-hook/plan.md +0 -469
- package/specs/post-tool-use-hook/spec.md +0 -364
- package/specs/private-tags/context.md +0 -288
- package/specs/private-tags/plan.md +0 -412
- package/specs/private-tags/spec.md +0 -345
- package/specs/progressive-disclosure/context.md +0 -346
- package/specs/progressive-disclosure/plan.md +0 -663
- package/specs/progressive-disclosure/spec.md +0 -415
- package/specs/selective-tool-observation/context.md +0 -100
- package/specs/selective-tool-observation/plan.md +0 -158
- package/specs/selective-tool-observation/spec.md +0 -127
- package/specs/task-entity-system/context.md +0 -297
- package/specs/task-entity-system/plan.md +0 -301
- package/specs/task-entity-system/spec.md +0 -314
- package/specs/thin-core-refactor/context.md +0 -275
- package/specs/thin-core-refactor/plan.md +0 -536
- package/specs/thin-core-refactor/spec.md +0 -465
- package/specs/vector-outbox-v2/context.md +0 -470
- package/specs/vector-outbox-v2/plan.md +0 -562
- package/specs/vector-outbox-v2/spec.md +0 -466
- package/specs/web-viewer-ui/context.md +0 -384
- package/specs/web-viewer-ui/plan.md +0 -797
- package/specs/web-viewer-ui/spec.md +0 -516
- package/src/adapters/claude/capture/index.ts +0 -3
- package/src/adapters/claude/context/index.ts +0 -3
- package/src/adapters/claude/hooks/index.ts +0 -21
- package/src/adapters/claude/hooks/post-tool-use.ts +0 -239
- package/src/adapters/claude/hooks/prompt-injection-policy.ts +0 -104
- package/src/adapters/claude/hooks/semantic-daemon-client.ts +0 -209
- package/src/adapters/claude/hooks/semantic-daemon.ts +0 -283
- package/src/adapters/claude/hooks/session-end.ts +0 -59
- package/src/adapters/claude/hooks/session-start.ts +0 -73
- package/src/adapters/claude/hooks/stop.ts +0 -128
- package/src/adapters/claude/hooks/user-prompt-submit.ts +0 -361
- package/src/adapters/claude/index.ts +0 -4
- package/src/adapters/claude/transcript/index.ts +0 -4
- package/src/adapters/claude/transcript/transcript-reader.ts +0 -57
- package/src/adapters/claude/transcript/turn-reconstructor.ts +0 -65
- package/src/apps/cli/claude-settings-hooks.ts +0 -138
- package/src/apps/cli/codex-import-runner.ts +0 -125
- package/src/apps/cli/codex-validation-output.ts +0 -95
- package/src/apps/cli/hermes-import-runner.ts +0 -130
- package/src/apps/cli/hermes-validation-output.ts +0 -91
- package/src/apps/cli/index.ts +0 -1735
- package/src/apps/cli/mcp-install.ts +0 -106
- package/src/apps/cli/retrieval-disclosure-output.ts +0 -196
- package/src/apps/dashboard/assets/js/bootstrap.js +0 -244
- package/src/apps/dashboard/assets/js/chat.js +0 -373
- package/src/apps/dashboard/assets/js/disclosure.js +0 -232
- package/src/apps/dashboard/assets/js/modals.js +0 -298
- package/src/apps/dashboard/assets/js/overview.js +0 -655
- package/src/apps/dashboard/assets/js/state.js +0 -72
- package/src/apps/dashboard/assets/js/views.js +0 -468
- package/src/apps/dashboard/index.html +0 -543
- package/src/apps/dashboard/index.ts +0 -3
- package/src/apps/dashboard/style.css +0 -1750
- package/src/apps/index.ts +0 -5
- package/src/apps/server/api/chat.ts +0 -244
- package/src/apps/server/api/citations.ts +0 -105
- package/src/apps/server/api/events.ts +0 -137
- package/src/apps/server/api/health.ts +0 -53
- package/src/apps/server/api/index.ts +0 -26
- package/src/apps/server/api/projects.ts +0 -74
- package/src/apps/server/api/search.ts +0 -184
- package/src/apps/server/api/sessions.ts +0 -115
- package/src/apps/server/api/stats.ts +0 -723
- package/src/apps/server/api/turns.ts +0 -143
- package/src/apps/server/api/utils.ts +0 -65
- package/src/apps/server/index.ts +0 -111
- package/src/cli/index.ts +0 -3
- package/src/cli/retrieval-disclosure-output.ts +0 -2
- package/src/compat/index.ts +0 -5
- package/src/core/canonical-key.ts +0 -186
- package/src/core/citation-generator.ts +0 -63
- package/src/core/consolidated-store.ts +0 -356
- package/src/core/consolidation-worker.ts +0 -493
- package/src/core/context-formatter.ts +0 -276
- package/src/core/continuity-manager.ts +0 -341
- package/src/core/db-wrapper.ts +0 -64
- package/src/core/derive/fact-deriver.ts +0 -170
- package/src/core/derive/index.ts +0 -2
- package/src/core/derive/summary-deriver.ts +0 -76
- package/src/core/edge-repo.ts +0 -333
- package/src/core/embedder.ts +0 -4
- package/src/core/engine/embedding-maintenance-service.ts +0 -187
- package/src/core/engine/endless-memory-services.ts +0 -4
- package/src/core/engine/index.ts +0 -19
- package/src/core/engine/memory-engine-services.ts +0 -170
- package/src/core/engine/memory-ingest-service.ts +0 -317
- package/src/core/engine/memory-query-service.ts +0 -173
- package/src/core/engine/memory-runtime-service.ts +0 -162
- package/src/core/engine/memory-service-composition.ts +0 -231
- package/src/core/engine/retrieval-analytics-service.ts +0 -181
- package/src/core/engine/retrieval-disclosure-service.ts +0 -420
- package/src/core/engine/retrieval-orchestrator.ts +0 -377
- package/src/core/engine/retrieval-services.ts +0 -176
- package/src/core/engine/shared-memory-services.ts +0 -4
- package/src/core/entity-repo.ts +0 -349
- package/src/core/event-store.ts +0 -779
- package/src/core/evidence-aligner.ts +0 -635
- package/src/core/external-market-context.ts +0 -582
- package/src/core/graduation-worker.ts +0 -171
- package/src/core/graduation.ts +0 -377
- package/src/core/index.ts +0 -64
- package/src/core/ingest-interceptor.ts +0 -80
- package/src/core/markdown-mirror.ts +0 -70
- package/src/core/matcher.ts +0 -208
- package/src/core/md-mirror.ts +0 -92
- package/src/core/metadata-extractor.ts +0 -203
- package/src/core/model/memory-fact.ts +0 -30
- package/src/core/model/memory-rule.ts +0 -14
- package/src/core/model/memory-summary.ts +0 -21
- package/src/core/model/raw-event.ts +0 -28
- package/src/core/model/retrieval-result.ts +0 -35
- package/src/core/mongo-sync-config.ts +0 -165
- package/src/core/mongo-sync-worker.ts +0 -381
- package/src/core/privacy/filter.ts +0 -190
- package/src/core/privacy/index.ts +0 -20
- package/src/core/privacy/tag-parser.ts +0 -145
- package/src/core/product-validation-matrix.ts +0 -314
- package/src/core/progressive-retriever.ts +0 -414
- package/src/core/registry/project-path.ts +0 -54
- package/src/core/registry/session-registry.ts +0 -69
- package/src/core/replay-evaluator.ts +0 -625
- package/src/core/retrieval-benchmark.ts +0 -117
- package/src/core/retrieval-quality.ts +0 -109
- package/src/core/retriever.ts +0 -800
- package/src/core/session-qrels.ts +0 -360
- package/src/core/shared-event-store.ts +0 -114
- package/src/core/shared-promoter.ts +0 -249
- package/src/core/shared-store.ts +0 -289
- package/src/core/shared-vector-store.ts +0 -203
- package/src/core/sqlite-event-store.ts +0 -1846
- package/src/core/sqlite-wrapper.ts +0 -116
- package/src/core/sync-worker.ts +0 -228
- package/src/core/tag-taxonomy.ts +0 -51
- package/src/core/task/blocker-resolver.ts +0 -333
- package/src/core/task/index.ts +0 -9
- package/src/core/task/task-matcher.ts +0 -240
- package/src/core/task/task-projector.ts +0 -358
- package/src/core/task/task-resolver.ts +0 -421
- package/src/core/turn-state.ts +0 -207
- package/src/core/types.ts +0 -952
- package/src/core/vector-outbox.ts +0 -299
- package/src/core/vector-store.ts +0 -231
- package/src/core/vector-worker.ts +0 -521
- package/src/core/working-set-store.ts +0 -257
- package/src/extensions/endless-memory/endless-memory-services.ts +0 -350
- package/src/extensions/endless-memory/index.ts +0 -1
- package/src/extensions/index.ts +0 -5
- package/src/extensions/mcp/handlers.ts +0 -960
- package/src/extensions/mcp/index.ts +0 -48
- package/src/extensions/mcp/tools.ts +0 -252
- package/src/extensions/shared-memory/index.ts +0 -1
- package/src/extensions/shared-memory/shared-memory-services.ts +0 -211
- package/src/extensions/vector/embedder.ts +0 -234
- package/src/extensions/vector/index.ts +0 -1
- package/src/hooks/post-tool-use.ts +0 -9
- package/src/hooks/semantic-daemon-client.ts +0 -1
- package/src/hooks/semantic-daemon.ts +0 -11
- package/src/hooks/session-end.ts +0 -9
- package/src/hooks/session-start.ts +0 -9
- package/src/hooks/stop.ts +0 -9
- package/src/hooks/user-prompt-submit.ts +0 -9
- package/src/index.ts +0 -13
- package/src/mcp/handlers.ts +0 -2
- package/src/mcp/index.ts +0 -4
- package/src/mcp/tools.ts +0 -2
- package/src/server/api/chat.ts +0 -2
- package/src/server/api/citations.ts +0 -2
- package/src/server/api/events.ts +0 -2
- package/src/server/api/health.ts +0 -2
- package/src/server/api/index.ts +0 -2
- package/src/server/api/projects.ts +0 -2
- package/src/server/api/search.ts +0 -2
- package/src/server/api/sessions.ts +0 -2
- package/src/server/api/stats.ts +0 -2
- package/src/server/api/turns.ts +0 -2
- package/src/server/api/utils.ts +0 -2
- package/src/server/index.ts +0 -2
- package/src/services/bootstrap-organizer.ts +0 -463
- package/src/services/codex-session-history-importer.ts +0 -966
- package/src/services/hermes-session-history-importer.ts +0 -733
- package/src/services/memory-service-config.ts +0 -36
- package/src/services/memory-service-registry.ts +0 -150
- package/src/services/memory-service.ts +0 -688
- package/src/services/session-history-importer.ts +0 -629
- package/tests/README.md +0 -23
- package/tests/adapters/claude/claude-semantic-daemon-adapter.test.ts +0 -54
- package/tests/adapters/claude/claude-transcript-reconstructor.test.ts +0 -98
- package/tests/adapters/claude-hook-prompt-injection-policy.test.ts +0 -99
- package/tests/apps/app-layer-boundary.test.ts +0 -48
- package/tests/apps/claude-settings-hooks.test.ts +0 -107
- package/tests/apps/cli-disclosure-output.test.ts +0 -212
- package/tests/apps/codex-import-runner.test.ts +0 -99
- package/tests/apps/codex-validation-output.test.ts +0 -100
- package/tests/apps/hermes-import-runner.test.ts +0 -99
- package/tests/apps/mcp-install-command.test.ts +0 -59
- package/tests/apps/package-build-entrypoints.test.ts +0 -30
- package/tests/apps/postinstall-embedding-backend.test.ts +0 -185
- package/tests/apps/search-api-disclosure.test.ts +0 -162
- package/tests/apps/stats-api-lightweight.test.ts +0 -67
- package/tests/apps/ui-disclosure-output.test.ts +0 -140
- package/tests/core/bootstrap-organizer.test.ts +0 -111
- package/tests/core/canonical-key.test.ts +0 -101
- package/tests/core/codex-session-history-importer-validation.test.ts +0 -185
- package/tests/core/consolidation-worker.test.ts +0 -75
- package/tests/core/embedding-maintenance-service.test.ts +0 -282
- package/tests/core/evidence-aligner.test.ts +0 -152
- package/tests/core/external-market-context.test.ts +0 -209
- package/tests/core/fact-deriver.test.ts +0 -79
- package/tests/core/hermes-session-history-importer-validation.test.ts +0 -609
- package/tests/core/ingest-interceptor.test.ts +0 -38
- package/tests/core/markdown-mirror.test.ts +0 -85
- package/tests/core/matcher.test.ts +0 -112
- package/tests/core/md-mirror.test.ts +0 -50
- package/tests/core/memory-engine-services.test.ts +0 -240
- package/tests/core/memory-ingest-service.test.ts +0 -296
- package/tests/core/memory-query-service.test.ts +0 -129
- package/tests/core/memory-runtime-service.test.ts +0 -201
- package/tests/core/memory-service-composition.test.ts +0 -192
- package/tests/core/memory-service-config.test.ts +0 -41
- package/tests/core/memory-service-facade.test.ts +0 -30
- package/tests/core/memory-service-registry.test.ts +0 -206
- package/tests/core/product-validation-matrix.test.ts +0 -61
- package/tests/core/project-registry.test.ts +0 -78
- package/tests/core/replay-evaluator.test.ts +0 -181
- package/tests/core/retrieval-analytics-service.test.ts +0 -210
- package/tests/core/retrieval-benchmark.test.ts +0 -93
- package/tests/core/retrieval-disclosure-service.test.ts +0 -264
- package/tests/core/retrieval-orchestrator.test.ts +0 -403
- package/tests/core/retrieval-quality.test.ts +0 -31
- package/tests/core/retrieval-services.test.ts +0 -185
- package/tests/core/retriever-fallback-chain.test.ts +0 -223
- package/tests/core/retriever-strategy-scope.test.ts +0 -164
- package/tests/core/retriever.memu-adoption.test.ts +0 -122
- package/tests/core/session-history-importer-filter.test.ts +0 -78
- package/tests/core/session-qrels.test.ts +0 -250
- package/tests/core/sqlite-event-store-replication.test.ts +0 -127
- package/tests/core/summary-deriver.test.ts +0 -66
- package/tests/extensions/embedder-warning-suppression.test.ts +0 -84
- package/tests/extensions/endless-memory-extension-boundary.test.ts +0 -17
- package/tests/extensions/endless-memory-services.test.ts +0 -325
- package/tests/extensions/mcp-context-tools.test.ts +0 -905
- package/tests/extensions/mcp-extension-boundary.test.ts +0 -21
- package/tests/extensions/mcp-package-build.test.ts +0 -22
- package/tests/extensions/mcp-project-aware-tools.test.ts +0 -102
- package/tests/extensions/shared-memory-extension-boundary.test.ts +0 -24
- package/tests/extensions/shared-memory-services.test.ts +0 -309
- package/tests/extensions/vector-extension-boundary.test.ts +0 -21
- package/tsconfig.json +0 -24
- package/vitest.config.ts +0 -15
|
@@ -1,966 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Codex Session History Importer
|
|
3
|
-
* Imports existing Codex CLI conversation history into memory
|
|
4
|
-
*
|
|
5
|
-
* Codex stores session history in:
|
|
6
|
-
* ~/.codex/sessions/YYYY/MM/DD/rollout-<timestamp>-<session-id>.jsonl
|
|
7
|
-
*/
|
|
8
|
-
|
|
9
|
-
import * as fs from 'fs';
|
|
10
|
-
import * as path from 'path';
|
|
11
|
-
import * as os from 'os';
|
|
12
|
-
import * as readline from 'readline';
|
|
13
|
-
import { createHash, randomUUID } from 'crypto';
|
|
14
|
-
import { MemoryService } from './memory-service.js';
|
|
15
|
-
import { registerSession } from '../core/registry/session-registry.js';
|
|
16
|
-
import type { ImportOptions, ImportResult } from './session-history-importer.js';
|
|
17
|
-
|
|
18
|
-
type CodexLogLine = {
|
|
19
|
-
timestamp?: string;
|
|
20
|
-
type?: string;
|
|
21
|
-
payload?: unknown;
|
|
22
|
-
};
|
|
23
|
-
|
|
24
|
-
type CodexSessionMetaPayload = {
|
|
25
|
-
id?: unknown;
|
|
26
|
-
cwd?: unknown;
|
|
27
|
-
timestamp?: unknown;
|
|
28
|
-
};
|
|
29
|
-
|
|
30
|
-
type CodexResponseItemMessagePayload = {
|
|
31
|
-
type?: unknown;
|
|
32
|
-
role?: unknown;
|
|
33
|
-
content?: unknown;
|
|
34
|
-
};
|
|
35
|
-
|
|
36
|
-
type CodexContentBlock = {
|
|
37
|
-
type?: unknown;
|
|
38
|
-
text?: unknown;
|
|
39
|
-
};
|
|
40
|
-
|
|
41
|
-
export const CODEX_VALIDATION_DEFAULT_MAX_CONTENT_CHARS = 10_000;
|
|
42
|
-
|
|
43
|
-
export interface CodexSessionMeta {
|
|
44
|
-
sessionId: string | null;
|
|
45
|
-
cwd: string | null;
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
export interface CodexValidationOptions {
|
|
49
|
-
sessionsDir?: string;
|
|
50
|
-
projectPath?: string;
|
|
51
|
-
limit?: number;
|
|
52
|
-
maxContentChars?: number;
|
|
53
|
-
anonymizeProjects?: boolean;
|
|
54
|
-
now?: Date;
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
export interface CodexValidationSource {
|
|
58
|
-
sessionsDir: string;
|
|
59
|
-
projectPath?: string;
|
|
60
|
-
projectFilterApplied: boolean;
|
|
61
|
-
sourcePaths: string[];
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
export interface CodexValidationLimits {
|
|
65
|
-
sessionLimit?: number;
|
|
66
|
-
maxContentChars: number;
|
|
67
|
-
}
|
|
68
|
-
|
|
69
|
-
export interface CodexValidationTotals {
|
|
70
|
-
sessionsScanned: number;
|
|
71
|
-
sessionsMatched: number;
|
|
72
|
-
filesRead: number;
|
|
73
|
-
recordsRead: number;
|
|
74
|
-
messagesNormalized: number;
|
|
75
|
-
turnsNormalized: number;
|
|
76
|
-
userMessages: number;
|
|
77
|
-
assistantMessages: number;
|
|
78
|
-
malformedLines: number;
|
|
79
|
-
skippedUnsupportedRecords: number;
|
|
80
|
-
emptyAssistantMessages: number;
|
|
81
|
-
truncatedMessages: number;
|
|
82
|
-
missingProjectCwd: number;
|
|
83
|
-
warnings: number;
|
|
84
|
-
}
|
|
85
|
-
|
|
86
|
-
export interface CodexProjectSummary {
|
|
87
|
-
projectHash: string;
|
|
88
|
-
pathLabel: string;
|
|
89
|
-
sessions: number;
|
|
90
|
-
messagesNormalized: number;
|
|
91
|
-
turnsNormalized: number;
|
|
92
|
-
userMessages: number;
|
|
93
|
-
assistantMessages: number;
|
|
94
|
-
malformedLines: number;
|
|
95
|
-
skippedUnsupportedRecords: number;
|
|
96
|
-
truncatedMessages: number;
|
|
97
|
-
emptyAssistantMessages: number;
|
|
98
|
-
}
|
|
99
|
-
|
|
100
|
-
export interface CodexSessionReplaySummary {
|
|
101
|
-
sessionId: string;
|
|
102
|
-
filePath: string;
|
|
103
|
-
projectHash: string;
|
|
104
|
-
pathLabel: string;
|
|
105
|
-
matched: boolean;
|
|
106
|
-
recordsRead: number;
|
|
107
|
-
messagesNormalized: number;
|
|
108
|
-
turnsNormalized: number;
|
|
109
|
-
userMessages: number;
|
|
110
|
-
assistantMessages: number;
|
|
111
|
-
malformedLines: number;
|
|
112
|
-
skippedUnsupportedRecords: number;
|
|
113
|
-
emptyAssistantMessages: number;
|
|
114
|
-
truncatedMessages: number;
|
|
115
|
-
missingProjectCwd: boolean;
|
|
116
|
-
warnings: string[];
|
|
117
|
-
}
|
|
118
|
-
|
|
119
|
-
export interface CodexSessionValidationReport {
|
|
120
|
-
generatedAt: string;
|
|
121
|
-
dryRun: true;
|
|
122
|
-
willMutate: false;
|
|
123
|
-
source: CodexValidationSource;
|
|
124
|
-
limits: CodexValidationLimits;
|
|
125
|
-
totals: CodexValidationTotals;
|
|
126
|
-
topProjects: CodexProjectSummary[];
|
|
127
|
-
sessions: CodexSessionReplaySummary[];
|
|
128
|
-
warnings: string[];
|
|
129
|
-
}
|
|
130
|
-
|
|
131
|
-
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
132
|
-
return typeof value === 'object' && value !== null;
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
function normalizeMaybeRealpath(p: string): string {
|
|
136
|
-
try {
|
|
137
|
-
return fs.realpathSync(p);
|
|
138
|
-
} catch {
|
|
139
|
-
return path.resolve(p);
|
|
140
|
-
}
|
|
141
|
-
}
|
|
142
|
-
|
|
143
|
-
interface CodexExtractedContent {
|
|
144
|
-
text: string | null;
|
|
145
|
-
originalLength: number;
|
|
146
|
-
truncated: boolean;
|
|
147
|
-
}
|
|
148
|
-
|
|
149
|
-
function extractCodexContentText(
|
|
150
|
-
content: unknown,
|
|
151
|
-
maxContentChars = CODEX_VALIDATION_DEFAULT_MAX_CONTENT_CHARS
|
|
152
|
-
): CodexExtractedContent {
|
|
153
|
-
const texts: string[] = [];
|
|
154
|
-
|
|
155
|
-
if (typeof content === 'string') {
|
|
156
|
-
if (content.length > 0) texts.push(content);
|
|
157
|
-
} else if (Array.isArray(content)) {
|
|
158
|
-
for (const block of content) {
|
|
159
|
-
if (!isRecord(block)) continue;
|
|
160
|
-
const b = block as CodexContentBlock;
|
|
161
|
-
const t = typeof b.type === 'string' ? b.type : '';
|
|
162
|
-
if (t !== 'input_text' && t !== 'output_text' && t !== 'text') continue;
|
|
163
|
-
if (typeof b.text === 'string' && b.text.length > 0) {
|
|
164
|
-
texts.push(b.text);
|
|
165
|
-
}
|
|
166
|
-
}
|
|
167
|
-
}
|
|
168
|
-
|
|
169
|
-
if (texts.length === 0) {
|
|
170
|
-
return { text: null, originalLength: 0, truncated: false };
|
|
171
|
-
}
|
|
172
|
-
|
|
173
|
-
const merged = texts.join('\n');
|
|
174
|
-
const truncated = merged.length > maxContentChars;
|
|
175
|
-
return {
|
|
176
|
-
text: truncated ? `${merged.slice(0, maxContentChars)}...[truncated]` : merged,
|
|
177
|
-
originalLength: merged.length,
|
|
178
|
-
truncated
|
|
179
|
-
};
|
|
180
|
-
}
|
|
181
|
-
|
|
182
|
-
function extractTextFromContent(content: unknown): string | null {
|
|
183
|
-
return extractCodexContentText(content).text;
|
|
184
|
-
}
|
|
185
|
-
|
|
186
|
-
export function getDefaultCodexSessionsDir(): string {
|
|
187
|
-
return path.join(os.homedir(), '.codex', 'sessions');
|
|
188
|
-
}
|
|
189
|
-
|
|
190
|
-
export function listCodexSessionFilesRecursive(rootDir: string): string[] {
|
|
191
|
-
if (!fs.existsSync(rootDir)) return [];
|
|
192
|
-
const out: string[] = [];
|
|
193
|
-
const stack: string[] = [rootDir];
|
|
194
|
-
|
|
195
|
-
while (stack.length > 0) {
|
|
196
|
-
const dir = stack.pop()!;
|
|
197
|
-
let entries: fs.Dirent[];
|
|
198
|
-
try {
|
|
199
|
-
entries = fs.readdirSync(dir, { withFileTypes: true });
|
|
200
|
-
} catch {
|
|
201
|
-
continue;
|
|
202
|
-
}
|
|
203
|
-
|
|
204
|
-
for (const ent of entries) {
|
|
205
|
-
const fullPath = path.join(dir, ent.name);
|
|
206
|
-
if (ent.isDirectory()) {
|
|
207
|
-
stack.push(fullPath);
|
|
208
|
-
} else if (ent.isFile() && ent.name.endsWith('.jsonl')) {
|
|
209
|
-
out.push(fullPath);
|
|
210
|
-
}
|
|
211
|
-
}
|
|
212
|
-
}
|
|
213
|
-
|
|
214
|
-
return out.sort();
|
|
215
|
-
}
|
|
216
|
-
|
|
217
|
-
function getFileMtimeMs(filePath: string): number {
|
|
218
|
-
try {
|
|
219
|
-
return fs.statSync(filePath).mtimeMs;
|
|
220
|
-
} catch {
|
|
221
|
-
return 0;
|
|
222
|
-
}
|
|
223
|
-
}
|
|
224
|
-
|
|
225
|
-
function selectRecentCodexSessionFiles(files: string[], sessionLimit?: number): string[] {
|
|
226
|
-
if (sessionLimit === undefined) return files;
|
|
227
|
-
const limit = Number.isFinite(sessionLimit) && sessionLimit > 0 ? Math.floor(sessionLimit) : undefined;
|
|
228
|
-
if (limit === undefined) return files;
|
|
229
|
-
return [...files]
|
|
230
|
-
.sort((a, b) => getFileMtimeMs(b) - getFileMtimeMs(a) || b.localeCompare(a))
|
|
231
|
-
.slice(0, limit);
|
|
232
|
-
}
|
|
233
|
-
|
|
234
|
-
function normalizePositiveImportLimit(limit?: number): number | undefined {
|
|
235
|
-
if (limit === undefined) return undefined;
|
|
236
|
-
return Number.isFinite(limit) && limit > 0 ? Math.floor(limit) : undefined;
|
|
237
|
-
}
|
|
238
|
-
|
|
239
|
-
function countStoredEntries(result: ImportResult): number {
|
|
240
|
-
return result.importedPrompts + result.importedResponses + result.skippedDuplicates;
|
|
241
|
-
}
|
|
242
|
-
|
|
243
|
-
export async function readCodexSessionMeta(filePath: string): Promise<CodexSessionMeta> {
|
|
244
|
-
const fileStream = fs.createReadStream(filePath, { encoding: 'utf-8' });
|
|
245
|
-
const rl = readline.createInterface({ input: fileStream, crlfDelay: Infinity });
|
|
246
|
-
|
|
247
|
-
try {
|
|
248
|
-
let linesRead = 0;
|
|
249
|
-
for await (const line of rl) {
|
|
250
|
-
linesRead++;
|
|
251
|
-
if (!line.trim()) continue;
|
|
252
|
-
try {
|
|
253
|
-
const obj = JSON.parse(line) as CodexLogLine;
|
|
254
|
-
if (obj.type !== 'session_meta') continue;
|
|
255
|
-
if (!isRecord(obj.payload)) break;
|
|
256
|
-
const payload = obj.payload as CodexSessionMetaPayload;
|
|
257
|
-
const sessionId = typeof payload.id === 'string' ? payload.id : null;
|
|
258
|
-
const cwd = typeof payload.cwd === 'string' ? payload.cwd : null;
|
|
259
|
-
return { sessionId, cwd };
|
|
260
|
-
} catch {
|
|
261
|
-
// Ignore malformed preamble lines while looking for session_meta.
|
|
262
|
-
}
|
|
263
|
-
|
|
264
|
-
// session_meta is expected near the top; do not scan huge transcripts twice.
|
|
265
|
-
if (linesRead >= 25) break;
|
|
266
|
-
}
|
|
267
|
-
} finally {
|
|
268
|
-
rl.close();
|
|
269
|
-
fileStream.close();
|
|
270
|
-
}
|
|
271
|
-
|
|
272
|
-
return { sessionId: null, cwd: null };
|
|
273
|
-
}
|
|
274
|
-
|
|
275
|
-
export function deriveCodexSessionIdFromFileName(filePath: string): string | null {
|
|
276
|
-
const base = path.basename(filePath, '.jsonl');
|
|
277
|
-
// Common: rollout-<date>-<uuid>
|
|
278
|
-
const m = base.match(/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$/i);
|
|
279
|
-
if (m?.[1]) return m[1];
|
|
280
|
-
return base.length > 0 ? base : null;
|
|
281
|
-
}
|
|
282
|
-
|
|
283
|
-
function createEmptyCodexValidationTotals(): CodexValidationTotals {
|
|
284
|
-
return {
|
|
285
|
-
sessionsScanned: 0,
|
|
286
|
-
sessionsMatched: 0,
|
|
287
|
-
filesRead: 0,
|
|
288
|
-
recordsRead: 0,
|
|
289
|
-
messagesNormalized: 0,
|
|
290
|
-
turnsNormalized: 0,
|
|
291
|
-
userMessages: 0,
|
|
292
|
-
assistantMessages: 0,
|
|
293
|
-
malformedLines: 0,
|
|
294
|
-
skippedUnsupportedRecords: 0,
|
|
295
|
-
emptyAssistantMessages: 0,
|
|
296
|
-
truncatedMessages: 0,
|
|
297
|
-
missingProjectCwd: 0,
|
|
298
|
-
warnings: 0
|
|
299
|
-
};
|
|
300
|
-
}
|
|
301
|
-
|
|
302
|
-
function projectHashFor(cwd: string | null): string {
|
|
303
|
-
return createHash('sha256').update(cwd ?? '(missing cwd)').digest('hex').slice(0, 12);
|
|
304
|
-
}
|
|
305
|
-
|
|
306
|
-
function projectPathLabel(cwd: string | null, anonymizeProjects: boolean): string {
|
|
307
|
-
const hash = projectHashFor(cwd);
|
|
308
|
-
if (anonymizeProjects) return cwd ? `project:${hash}` : `project:${hash}:missing-cwd`;
|
|
309
|
-
return cwd ?? '(missing cwd)';
|
|
310
|
-
}
|
|
311
|
-
|
|
312
|
-
function isProjectMatch(cwd: string | null, projectPath?: string): boolean {
|
|
313
|
-
if (!projectPath) return true;
|
|
314
|
-
if (!cwd) return false;
|
|
315
|
-
return normalizeMaybeRealpath(cwd) === normalizeMaybeRealpath(projectPath);
|
|
316
|
-
}
|
|
317
|
-
|
|
318
|
-
function addSessionToProject(projects: Map<string, CodexProjectSummary>, session: CodexSessionReplaySummary): void {
|
|
319
|
-
const existing = projects.get(session.projectHash) ?? {
|
|
320
|
-
projectHash: session.projectHash,
|
|
321
|
-
pathLabel: session.pathLabel,
|
|
322
|
-
sessions: 0,
|
|
323
|
-
messagesNormalized: 0,
|
|
324
|
-
turnsNormalized: 0,
|
|
325
|
-
userMessages: 0,
|
|
326
|
-
assistantMessages: 0,
|
|
327
|
-
malformedLines: 0,
|
|
328
|
-
skippedUnsupportedRecords: 0,
|
|
329
|
-
truncatedMessages: 0,
|
|
330
|
-
emptyAssistantMessages: 0
|
|
331
|
-
};
|
|
332
|
-
|
|
333
|
-
existing.sessions += 1;
|
|
334
|
-
existing.messagesNormalized += session.messagesNormalized;
|
|
335
|
-
existing.turnsNormalized += session.turnsNormalized;
|
|
336
|
-
existing.userMessages += session.userMessages;
|
|
337
|
-
existing.assistantMessages += session.assistantMessages;
|
|
338
|
-
existing.malformedLines += session.malformedLines;
|
|
339
|
-
existing.skippedUnsupportedRecords += session.skippedUnsupportedRecords;
|
|
340
|
-
existing.truncatedMessages += session.truncatedMessages;
|
|
341
|
-
existing.emptyAssistantMessages += session.emptyAssistantMessages;
|
|
342
|
-
projects.set(session.projectHash, existing);
|
|
343
|
-
}
|
|
344
|
-
|
|
345
|
-
function addSessionToTotals(totals: CodexValidationTotals, session: CodexSessionReplaySummary): void {
|
|
346
|
-
totals.filesRead += 1;
|
|
347
|
-
totals.recordsRead += session.recordsRead;
|
|
348
|
-
totals.messagesNormalized += session.messagesNormalized;
|
|
349
|
-
totals.turnsNormalized += session.turnsNormalized;
|
|
350
|
-
totals.userMessages += session.userMessages;
|
|
351
|
-
totals.assistantMessages += session.assistantMessages;
|
|
352
|
-
totals.malformedLines += session.malformedLines;
|
|
353
|
-
totals.skippedUnsupportedRecords += session.skippedUnsupportedRecords;
|
|
354
|
-
totals.emptyAssistantMessages += session.emptyAssistantMessages;
|
|
355
|
-
totals.truncatedMessages += session.truncatedMessages;
|
|
356
|
-
}
|
|
357
|
-
|
|
358
|
-
export async function normalizeCodexSessionFile(
|
|
359
|
-
filePath: string,
|
|
360
|
-
options: {
|
|
361
|
-
meta?: CodexSessionMeta;
|
|
362
|
-
matched?: boolean;
|
|
363
|
-
maxContentChars?: number;
|
|
364
|
-
anonymizeProjects?: boolean;
|
|
365
|
-
} = {}
|
|
366
|
-
): Promise<CodexSessionReplaySummary> {
|
|
367
|
-
const meta = options.meta ?? await readCodexSessionMeta(filePath);
|
|
368
|
-
const sessionId = meta.sessionId ?? deriveCodexSessionIdFromFileName(filePath) ?? path.basename(filePath, '.jsonl');
|
|
369
|
-
const projectHash = projectHashFor(meta.cwd);
|
|
370
|
-
const pathLabel = projectPathLabel(meta.cwd, options.anonymizeProjects === true);
|
|
371
|
-
const summary: CodexSessionReplaySummary = {
|
|
372
|
-
sessionId,
|
|
373
|
-
filePath,
|
|
374
|
-
projectHash,
|
|
375
|
-
pathLabel,
|
|
376
|
-
matched: options.matched ?? true,
|
|
377
|
-
recordsRead: 0,
|
|
378
|
-
messagesNormalized: 0,
|
|
379
|
-
turnsNormalized: 0,
|
|
380
|
-
userMessages: 0,
|
|
381
|
-
assistantMessages: 0,
|
|
382
|
-
malformedLines: 0,
|
|
383
|
-
skippedUnsupportedRecords: 0,
|
|
384
|
-
emptyAssistantMessages: 0,
|
|
385
|
-
truncatedMessages: 0,
|
|
386
|
-
missingProjectCwd: !meta.cwd,
|
|
387
|
-
warnings: []
|
|
388
|
-
};
|
|
389
|
-
|
|
390
|
-
if (!meta.cwd) {
|
|
391
|
-
summary.warnings.push('session_meta missing cwd; project matching is unavailable for this session');
|
|
392
|
-
}
|
|
393
|
-
|
|
394
|
-
const maxContentChars = options.maxContentChars ?? CODEX_VALIDATION_DEFAULT_MAX_CONTENT_CHARS;
|
|
395
|
-
const fileStream = fs.createReadStream(filePath, { encoding: 'utf-8' });
|
|
396
|
-
const rl = readline.createInterface({ input: fileStream, crlfDelay: Infinity });
|
|
397
|
-
|
|
398
|
-
try {
|
|
399
|
-
for await (const line of rl) {
|
|
400
|
-
if (!line.trim()) continue;
|
|
401
|
-
summary.recordsRead += 1;
|
|
402
|
-
|
|
403
|
-
let entry: CodexLogLine;
|
|
404
|
-
try {
|
|
405
|
-
entry = JSON.parse(line) as CodexLogLine;
|
|
406
|
-
} catch {
|
|
407
|
-
summary.malformedLines += 1;
|
|
408
|
-
continue;
|
|
409
|
-
}
|
|
410
|
-
|
|
411
|
-
if (entry.type === 'session_meta') continue;
|
|
412
|
-
if (entry.type !== 'response_item' || !isRecord(entry.payload)) {
|
|
413
|
-
summary.skippedUnsupportedRecords += 1;
|
|
414
|
-
continue;
|
|
415
|
-
}
|
|
416
|
-
|
|
417
|
-
const payload = entry.payload as CodexResponseItemMessagePayload;
|
|
418
|
-
if (payload.type !== 'message') {
|
|
419
|
-
summary.skippedUnsupportedRecords += 1;
|
|
420
|
-
continue;
|
|
421
|
-
}
|
|
422
|
-
|
|
423
|
-
const role = typeof payload.role === 'string' ? payload.role : null;
|
|
424
|
-
if (role !== 'user' && role !== 'assistant') {
|
|
425
|
-
summary.skippedUnsupportedRecords += 1;
|
|
426
|
-
continue;
|
|
427
|
-
}
|
|
428
|
-
|
|
429
|
-
const extracted = extractCodexContentText(payload.content, maxContentChars);
|
|
430
|
-
if (!extracted.text) {
|
|
431
|
-
if (role === 'assistant') {
|
|
432
|
-
summary.emptyAssistantMessages += 1;
|
|
433
|
-
} else {
|
|
434
|
-
summary.skippedUnsupportedRecords += 1;
|
|
435
|
-
}
|
|
436
|
-
continue;
|
|
437
|
-
}
|
|
438
|
-
|
|
439
|
-
if (extracted.truncated) {
|
|
440
|
-
summary.truncatedMessages += 1;
|
|
441
|
-
}
|
|
442
|
-
|
|
443
|
-
summary.messagesNormalized += 1;
|
|
444
|
-
if (role === 'user') {
|
|
445
|
-
summary.userMessages += 1;
|
|
446
|
-
summary.turnsNormalized += 1;
|
|
447
|
-
} else {
|
|
448
|
-
summary.assistantMessages += 1;
|
|
449
|
-
}
|
|
450
|
-
}
|
|
451
|
-
} finally {
|
|
452
|
-
rl.close();
|
|
453
|
-
fileStream.close();
|
|
454
|
-
}
|
|
455
|
-
|
|
456
|
-
return summary;
|
|
457
|
-
}
|
|
458
|
-
|
|
459
|
-
export async function validateCodexSessions(options: CodexValidationOptions = {}): Promise<CodexSessionValidationReport> {
|
|
460
|
-
const sessionsDir = path.resolve(options.sessionsDir ?? getDefaultCodexSessionsDir());
|
|
461
|
-
const maxContentChars = options.maxContentChars ?? CODEX_VALIDATION_DEFAULT_MAX_CONTENT_CHARS;
|
|
462
|
-
const report: CodexSessionValidationReport = {
|
|
463
|
-
generatedAt: (options.now ?? new Date()).toISOString(),
|
|
464
|
-
dryRun: true,
|
|
465
|
-
willMutate: false,
|
|
466
|
-
source: {
|
|
467
|
-
sessionsDir,
|
|
468
|
-
projectPath: options.projectPath,
|
|
469
|
-
projectFilterApplied: Boolean(options.projectPath),
|
|
470
|
-
sourcePaths: [sessionsDir]
|
|
471
|
-
},
|
|
472
|
-
limits: {
|
|
473
|
-
sessionLimit: options.limit,
|
|
474
|
-
maxContentChars
|
|
475
|
-
},
|
|
476
|
-
totals: createEmptyCodexValidationTotals(),
|
|
477
|
-
topProjects: [],
|
|
478
|
-
sessions: [],
|
|
479
|
-
warnings: []
|
|
480
|
-
};
|
|
481
|
-
|
|
482
|
-
if (!fs.existsSync(sessionsDir)) {
|
|
483
|
-
report.warnings.push(`Codex sessions directory not found: ${sessionsDir}`);
|
|
484
|
-
report.totals.warnings = report.warnings.length;
|
|
485
|
-
return report;
|
|
486
|
-
}
|
|
487
|
-
|
|
488
|
-
const sessionFiles = listCodexSessionFilesRecursive(sessionsDir);
|
|
489
|
-
const limitedFiles = typeof options.limit === 'number' && Number.isFinite(options.limit) && options.limit > 0
|
|
490
|
-
? sessionFiles.slice(0, Math.floor(options.limit))
|
|
491
|
-
: sessionFiles;
|
|
492
|
-
const projects = new Map<string, CodexProjectSummary>();
|
|
493
|
-
|
|
494
|
-
for (const filePath of limitedFiles) {
|
|
495
|
-
const meta = await readCodexSessionMeta(filePath);
|
|
496
|
-
report.totals.sessionsScanned += 1;
|
|
497
|
-
if (!meta.cwd) {
|
|
498
|
-
report.totals.missingProjectCwd += 1;
|
|
499
|
-
}
|
|
500
|
-
|
|
501
|
-
const matched = isProjectMatch(meta.cwd, options.projectPath);
|
|
502
|
-
if (!matched) continue;
|
|
503
|
-
|
|
504
|
-
report.totals.sessionsMatched += 1;
|
|
505
|
-
const sessionSummary = await normalizeCodexSessionFile(filePath, {
|
|
506
|
-
meta,
|
|
507
|
-
matched,
|
|
508
|
-
maxContentChars,
|
|
509
|
-
anonymizeProjects: options.anonymizeProjects
|
|
510
|
-
});
|
|
511
|
-
report.sessions.push(sessionSummary);
|
|
512
|
-
addSessionToTotals(report.totals, sessionSummary);
|
|
513
|
-
addSessionToProject(projects, sessionSummary);
|
|
514
|
-
}
|
|
515
|
-
|
|
516
|
-
report.topProjects = [...projects.values()].sort((a, b) => {
|
|
517
|
-
const byMessages = b.messagesNormalized - a.messagesNormalized;
|
|
518
|
-
if (byMessages !== 0) return byMessages;
|
|
519
|
-
return b.sessions - a.sessions;
|
|
520
|
-
}).slice(0, 10);
|
|
521
|
-
|
|
522
|
-
if (report.totals.missingProjectCwd > 0) {
|
|
523
|
-
report.warnings.push(`${report.totals.missingProjectCwd} session(s) missing cwd; project matching only uses sessions with cwd`);
|
|
524
|
-
}
|
|
525
|
-
if (report.totals.malformedLines > 0) {
|
|
526
|
-
report.warnings.push(`${report.totals.malformedLines} malformed JSONL line(s) skipped`);
|
|
527
|
-
}
|
|
528
|
-
if (options.projectPath && report.totals.sessionsMatched === 0) {
|
|
529
|
-
report.warnings.push(`No Codex sessions matched project cwd: ${options.projectPath}`);
|
|
530
|
-
}
|
|
531
|
-
report.totals.warnings = report.warnings.length;
|
|
532
|
-
|
|
533
|
-
return report;
|
|
534
|
-
}
|
|
535
|
-
|
|
536
|
-
export interface CodexSessionHistoryImporterOptions {
|
|
537
|
-
sessionsDir?: string;
|
|
538
|
-
}
|
|
539
|
-
|
|
540
|
-
export class CodexSessionHistoryImporter {
|
|
541
|
-
private readonly memoryService: MemoryService;
|
|
542
|
-
private readonly sessionsRoot: string;
|
|
543
|
-
|
|
544
|
-
constructor(memoryService: MemoryService, options: CodexSessionHistoryImporterOptions = {}) {
|
|
545
|
-
this.memoryService = memoryService;
|
|
546
|
-
this.sessionsRoot = options.sessionsDir
|
|
547
|
-
? path.resolve(options.sessionsDir)
|
|
548
|
-
: path.join(os.homedir(), '.codex', 'sessions');
|
|
549
|
-
}
|
|
550
|
-
|
|
551
|
-
private getSessionsRoot(): string {
|
|
552
|
-
return this.sessionsRoot;
|
|
553
|
-
}
|
|
554
|
-
|
|
555
|
-
private listSessionFilesRecursive(rootDir: string): string[] {
|
|
556
|
-
if (!fs.existsSync(rootDir)) return [];
|
|
557
|
-
const out: string[] = [];
|
|
558
|
-
const stack: string[] = [rootDir];
|
|
559
|
-
|
|
560
|
-
while (stack.length > 0) {
|
|
561
|
-
const dir = stack.pop()!;
|
|
562
|
-
let entries: fs.Dirent[];
|
|
563
|
-
try {
|
|
564
|
-
entries = fs.readdirSync(dir, { withFileTypes: true });
|
|
565
|
-
} catch {
|
|
566
|
-
continue;
|
|
567
|
-
}
|
|
568
|
-
|
|
569
|
-
for (const ent of entries) {
|
|
570
|
-
const fullPath = path.join(dir, ent.name);
|
|
571
|
-
if (ent.isDirectory()) {
|
|
572
|
-
stack.push(fullPath);
|
|
573
|
-
} else if (ent.isFile() && ent.name.endsWith('.jsonl')) {
|
|
574
|
-
out.push(fullPath);
|
|
575
|
-
}
|
|
576
|
-
}
|
|
577
|
-
}
|
|
578
|
-
|
|
579
|
-
return out;
|
|
580
|
-
}
|
|
581
|
-
|
|
582
|
-
private async readSessionMeta(filePath: string): Promise<{ sessionId: string | null; cwd: string | null }> {
|
|
583
|
-
const fileStream = fs.createReadStream(filePath, { encoding: 'utf-8' });
|
|
584
|
-
const rl = readline.createInterface({ input: fileStream, crlfDelay: Infinity });
|
|
585
|
-
|
|
586
|
-
try {
|
|
587
|
-
let linesRead = 0;
|
|
588
|
-
for await (const line of rl) {
|
|
589
|
-
linesRead++;
|
|
590
|
-
if (!line.trim()) continue;
|
|
591
|
-
try {
|
|
592
|
-
const obj = JSON.parse(line) as CodexLogLine;
|
|
593
|
-
if (obj.type !== 'session_meta') continue;
|
|
594
|
-
if (!isRecord(obj.payload)) break;
|
|
595
|
-
const payload = obj.payload as CodexSessionMetaPayload;
|
|
596
|
-
const sessionId = typeof payload.id === 'string' ? payload.id : null;
|
|
597
|
-
const cwd = typeof payload.cwd === 'string' ? payload.cwd : null;
|
|
598
|
-
return { sessionId, cwd };
|
|
599
|
-
} catch {
|
|
600
|
-
// ignore parse errors; keep scanning initial lines
|
|
601
|
-
}
|
|
602
|
-
|
|
603
|
-
// session_meta is expected at the top; don't scan entire file.
|
|
604
|
-
if (linesRead >= 25) break;
|
|
605
|
-
}
|
|
606
|
-
} finally {
|
|
607
|
-
rl.close();
|
|
608
|
-
fileStream.close();
|
|
609
|
-
}
|
|
610
|
-
|
|
611
|
-
return { sessionId: null, cwd: null };
|
|
612
|
-
}
|
|
613
|
-
|
|
614
|
-
private deriveSessionIdFromFileName(filePath: string): string | null {
|
|
615
|
-
const base = path.basename(filePath, '.jsonl');
|
|
616
|
-
// Common: rollout-<date>-<uuid>
|
|
617
|
-
const m = base.match(/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$/i);
|
|
618
|
-
if (m?.[1]) return m[1];
|
|
619
|
-
return base.length > 0 ? base : null;
|
|
620
|
-
}
|
|
621
|
-
|
|
622
|
-
async importProject(projectPath: string, options: ImportOptions = {}): Promise<ImportResult> {
|
|
623
|
-
const result: ImportResult = {
|
|
624
|
-
totalSessions: 0,
|
|
625
|
-
totalMessages: 0,
|
|
626
|
-
importedPrompts: 0,
|
|
627
|
-
importedResponses: 0,
|
|
628
|
-
skippedDuplicates: 0,
|
|
629
|
-
errors: []
|
|
630
|
-
};
|
|
631
|
-
|
|
632
|
-
const onProgress = options.onProgress;
|
|
633
|
-
const sessionsRoot = this.getSessionsRoot();
|
|
634
|
-
if (!fs.existsSync(sessionsRoot)) {
|
|
635
|
-
result.errors.push(`Codex sessions directory not found: ${sessionsRoot}`);
|
|
636
|
-
return result;
|
|
637
|
-
}
|
|
638
|
-
|
|
639
|
-
const normalizedTarget = normalizeMaybeRealpath(projectPath);
|
|
640
|
-
onProgress?.({ phase: 'scan', message: 'Scanning Codex session files...' });
|
|
641
|
-
|
|
642
|
-
const sessionFiles = this.listSessionFilesRecursive(sessionsRoot);
|
|
643
|
-
const matchingFiles: string[] = [];
|
|
644
|
-
|
|
645
|
-
// Filter by original CWD stored in session_meta.payload.cwd
|
|
646
|
-
for (const filePath of sessionFiles) {
|
|
647
|
-
try {
|
|
648
|
-
const meta = await this.readSessionMeta(filePath);
|
|
649
|
-
if (!meta.cwd) continue;
|
|
650
|
-
if (normalizeMaybeRealpath(meta.cwd) === normalizedTarget) {
|
|
651
|
-
matchingFiles.push(filePath);
|
|
652
|
-
}
|
|
653
|
-
} catch {
|
|
654
|
-
// ignore
|
|
655
|
-
}
|
|
656
|
-
}
|
|
657
|
-
|
|
658
|
-
const selectedFiles = selectRecentCodexSessionFiles(matchingFiles, options.sessionLimit);
|
|
659
|
-
onProgress?.({ phase: 'scan', message: `Found ${matchingFiles.length} Codex session(s) for this project` });
|
|
660
|
-
|
|
661
|
-
const effectiveProjectPath = options.projectPath ?? projectPath;
|
|
662
|
-
const totalLimit = normalizePositiveImportLimit(options.limit);
|
|
663
|
-
let storedAcrossSessions = 0;
|
|
664
|
-
|
|
665
|
-
for (let i = 0; i < selectedFiles.length; i++) {
|
|
666
|
-
if (totalLimit !== undefined && storedAcrossSessions >= totalLimit) break;
|
|
667
|
-
const filePath = selectedFiles[i];
|
|
668
|
-
const remainingLimit = totalLimit === undefined ? undefined : totalLimit - storedAcrossSessions;
|
|
669
|
-
try {
|
|
670
|
-
onProgress?.({ phase: 'session-start', sessionIndex: i, totalSessions: selectedFiles.length, filePath });
|
|
671
|
-
const sessionResult = await this.importSessionFile(filePath, {
|
|
672
|
-
...options,
|
|
673
|
-
limit: remainingLimit,
|
|
674
|
-
projectPath: effectiveProjectPath,
|
|
675
|
-
_sessionIndex: i,
|
|
676
|
-
} as ImportOptions & { _sessionIndex: number });
|
|
677
|
-
|
|
678
|
-
result.totalSessions++;
|
|
679
|
-
result.totalMessages += sessionResult.totalMessages;
|
|
680
|
-
result.importedPrompts += sessionResult.importedPrompts;
|
|
681
|
-
result.importedResponses += sessionResult.importedResponses;
|
|
682
|
-
result.skippedDuplicates += sessionResult.skippedDuplicates;
|
|
683
|
-
result.errors.push(...sessionResult.errors);
|
|
684
|
-
storedAcrossSessions += countStoredEntries(sessionResult);
|
|
685
|
-
|
|
686
|
-
onProgress?.({
|
|
687
|
-
phase: 'session-done',
|
|
688
|
-
sessionIndex: i,
|
|
689
|
-
importedPrompts: sessionResult.importedPrompts,
|
|
690
|
-
importedResponses: sessionResult.importedResponses,
|
|
691
|
-
skipped: sessionResult.skippedDuplicates
|
|
692
|
-
});
|
|
693
|
-
} catch (error) {
|
|
694
|
-
result.errors.push(`Failed to import ${filePath}: ${error}`);
|
|
695
|
-
}
|
|
696
|
-
}
|
|
697
|
-
|
|
698
|
-
return result;
|
|
699
|
-
}
|
|
700
|
-
|
|
701
|
-
async importAll(options: ImportOptions = {}): Promise<ImportResult> {
|
|
702
|
-
const result: ImportResult = {
|
|
703
|
-
totalSessions: 0,
|
|
704
|
-
totalMessages: 0,
|
|
705
|
-
importedPrompts: 0,
|
|
706
|
-
importedResponses: 0,
|
|
707
|
-
skippedDuplicates: 0,
|
|
708
|
-
errors: []
|
|
709
|
-
};
|
|
710
|
-
|
|
711
|
-
const onProgress = options.onProgress;
|
|
712
|
-
const sessionsRoot = this.getSessionsRoot();
|
|
713
|
-
if (!fs.existsSync(sessionsRoot)) {
|
|
714
|
-
result.errors.push(`Codex sessions directory not found: ${sessionsRoot}`);
|
|
715
|
-
return result;
|
|
716
|
-
}
|
|
717
|
-
|
|
718
|
-
onProgress?.({ phase: 'scan', message: 'Scanning all Codex sessions...' });
|
|
719
|
-
const sessionFiles = this.listSessionFilesRecursive(sessionsRoot);
|
|
720
|
-
const selectedFiles = selectRecentCodexSessionFiles(sessionFiles, options.sessionLimit);
|
|
721
|
-
onProgress?.({ phase: 'scan', message: `Found ${sessionFiles.length} Codex session file(s)` });
|
|
722
|
-
|
|
723
|
-
const totalLimit = normalizePositiveImportLimit(options.limit);
|
|
724
|
-
let storedAcrossSessions = 0;
|
|
725
|
-
|
|
726
|
-
for (let i = 0; i < selectedFiles.length; i++) {
|
|
727
|
-
if (totalLimit !== undefined && storedAcrossSessions >= totalLimit) break;
|
|
728
|
-
const filePath = selectedFiles[i];
|
|
729
|
-
const remainingLimit = totalLimit === undefined ? undefined : totalLimit - storedAcrossSessions;
|
|
730
|
-
try {
|
|
731
|
-
onProgress?.({ phase: 'session-start', sessionIndex: i, totalSessions: selectedFiles.length, filePath });
|
|
732
|
-
const sessionResult = await this.importSessionFile(filePath, {
|
|
733
|
-
...options,
|
|
734
|
-
limit: remainingLimit,
|
|
735
|
-
_sessionIndex: i,
|
|
736
|
-
} as ImportOptions & { _sessionIndex: number });
|
|
737
|
-
|
|
738
|
-
result.totalSessions++;
|
|
739
|
-
result.totalMessages += sessionResult.totalMessages;
|
|
740
|
-
result.importedPrompts += sessionResult.importedPrompts;
|
|
741
|
-
result.importedResponses += sessionResult.importedResponses;
|
|
742
|
-
result.skippedDuplicates += sessionResult.skippedDuplicates;
|
|
743
|
-
result.errors.push(...sessionResult.errors);
|
|
744
|
-
storedAcrossSessions += countStoredEntries(sessionResult);
|
|
745
|
-
|
|
746
|
-
onProgress?.({
|
|
747
|
-
phase: 'session-done',
|
|
748
|
-
sessionIndex: i,
|
|
749
|
-
importedPrompts: sessionResult.importedPrompts,
|
|
750
|
-
importedResponses: sessionResult.importedResponses,
|
|
751
|
-
skipped: sessionResult.skippedDuplicates
|
|
752
|
-
});
|
|
753
|
-
} catch (error) {
|
|
754
|
-
result.errors.push(`Failed to process ${filePath}: ${error}`);
|
|
755
|
-
}
|
|
756
|
-
}
|
|
757
|
-
|
|
758
|
-
return result;
|
|
759
|
-
}
|
|
760
|
-
|
|
761
|
-
async importSessionFile(filePath: string, options: ImportOptions = {}): Promise<ImportResult> {
|
|
762
|
-
const result: ImportResult = {
|
|
763
|
-
totalSessions: 1,
|
|
764
|
-
totalMessages: 0,
|
|
765
|
-
importedPrompts: 0,
|
|
766
|
-
importedResponses: 0,
|
|
767
|
-
skippedDuplicates: 0,
|
|
768
|
-
errors: []
|
|
769
|
-
};
|
|
770
|
-
|
|
771
|
-
if (!fs.existsSync(filePath)) {
|
|
772
|
-
result.errors.push(`File not found: ${filePath}`);
|
|
773
|
-
return result;
|
|
774
|
-
}
|
|
775
|
-
|
|
776
|
-
const meta = await this.readSessionMeta(filePath);
|
|
777
|
-
const sessionId = meta.sessionId ?? this.deriveSessionIdFromFileName(filePath);
|
|
778
|
-
|
|
779
|
-
if (!sessionId) {
|
|
780
|
-
result.errors.push(`Could not determine session id for: ${filePath}`);
|
|
781
|
-
return result;
|
|
782
|
-
}
|
|
783
|
-
|
|
784
|
-
const effectiveProjectPath = options.projectPath ?? meta.cwd ?? undefined;
|
|
785
|
-
|
|
786
|
-
if (options.force) {
|
|
787
|
-
const deleted = await this.memoryService.deleteSessionEvents(sessionId);
|
|
788
|
-
if (options.verbose && deleted > 0) {
|
|
789
|
-
console.log(` Deleted ${deleted} existing events for session ${sessionId}`);
|
|
790
|
-
}
|
|
791
|
-
}
|
|
792
|
-
|
|
793
|
-
await this.memoryService.startSession(sessionId, effectiveProjectPath);
|
|
794
|
-
|
|
795
|
-
const fileStream = fs.createReadStream(filePath, { encoding: 'utf-8' });
|
|
796
|
-
const rl = readline.createInterface({ input: fileStream, crlfDelay: Infinity });
|
|
797
|
-
|
|
798
|
-
const onProgress = options.onProgress;
|
|
799
|
-
const sessionIndex = (options as ImportOptions & { _sessionIndex?: number })._sessionIndex ?? 0;
|
|
800
|
-
let lastProgressAt = 0;
|
|
801
|
-
const limit = options.limit || Infinity;
|
|
802
|
-
let storedCount = 0;
|
|
803
|
-
|
|
804
|
-
let currentTurnId: string | null = null;
|
|
805
|
-
let textBuffer: string[] = [];
|
|
806
|
-
let lastTimestamp: string | undefined;
|
|
807
|
-
|
|
808
|
-
const flushTextBuffer = async () => {
|
|
809
|
-
if (storedCount >= limit) { textBuffer = []; return; }
|
|
810
|
-
if (textBuffer.length === 0 || !currentTurnId) return;
|
|
811
|
-
|
|
812
|
-
const substantive = textBuffer.filter(t => t.length >= 100);
|
|
813
|
-
const merged = substantive.length > 0
|
|
814
|
-
? substantive.join('\n\n')
|
|
815
|
-
: textBuffer.reduce((a, b) => a.length >= b.length ? a : b, '');
|
|
816
|
-
|
|
817
|
-
if (!merged) { textBuffer = []; return; }
|
|
818
|
-
|
|
819
|
-
const truncated = merged.length > 10000
|
|
820
|
-
? merged.slice(0, 10000) + '...[truncated]'
|
|
821
|
-
: merged;
|
|
822
|
-
|
|
823
|
-
const appendResult = await this.memoryService.storeAgentResponse(
|
|
824
|
-
sessionId,
|
|
825
|
-
truncated,
|
|
826
|
-
{ importedFrom: filePath, originalTimestamp: lastTimestamp, turnId: currentTurnId, source: 'codex' }
|
|
827
|
-
);
|
|
828
|
-
|
|
829
|
-
if (appendResult.success && appendResult.isDuplicate) {
|
|
830
|
-
result.skippedDuplicates++;
|
|
831
|
-
} else {
|
|
832
|
-
result.importedResponses++;
|
|
833
|
-
}
|
|
834
|
-
storedCount++;
|
|
835
|
-
textBuffer = [];
|
|
836
|
-
};
|
|
837
|
-
|
|
838
|
-
try {
|
|
839
|
-
for await (const line of rl) {
|
|
840
|
-
if (storedCount >= limit) break;
|
|
841
|
-
try {
|
|
842
|
-
const entry = JSON.parse(line) as CodexLogLine;
|
|
843
|
-
result.totalMessages++;
|
|
844
|
-
|
|
845
|
-
if (entry.type === 'response_item' && isRecord(entry.payload)) {
|
|
846
|
-
const payload = entry.payload as CodexResponseItemMessagePayload;
|
|
847
|
-
if (payload.type !== 'message') continue;
|
|
848
|
-
|
|
849
|
-
const role = typeof payload.role === 'string' ? payload.role : null;
|
|
850
|
-
if (!role) continue;
|
|
851
|
-
|
|
852
|
-
if (role === 'user') {
|
|
853
|
-
await flushTextBuffer();
|
|
854
|
-
|
|
855
|
-
const content = extractTextFromContent(payload.content);
|
|
856
|
-
if (!content) continue;
|
|
857
|
-
|
|
858
|
-
currentTurnId = randomUUID();
|
|
859
|
-
|
|
860
|
-
const appendResult = await this.memoryService.storeUserPrompt(
|
|
861
|
-
sessionId,
|
|
862
|
-
content,
|
|
863
|
-
{ importedFrom: filePath, originalTimestamp: entry.timestamp, turnId: currentTurnId, source: 'codex' }
|
|
864
|
-
);
|
|
865
|
-
|
|
866
|
-
if (appendResult.success && appendResult.isDuplicate) {
|
|
867
|
-
result.skippedDuplicates++;
|
|
868
|
-
} else {
|
|
869
|
-
result.importedPrompts++;
|
|
870
|
-
}
|
|
871
|
-
storedCount++;
|
|
872
|
-
} else if (role === 'assistant') {
|
|
873
|
-
const content = extractTextFromContent(payload.content);
|
|
874
|
-
if (content) {
|
|
875
|
-
textBuffer.push(content);
|
|
876
|
-
if (typeof entry.timestamp === 'string') {
|
|
877
|
-
lastTimestamp = entry.timestamp;
|
|
878
|
-
}
|
|
879
|
-
}
|
|
880
|
-
} else {
|
|
881
|
-
// developer/system/tool messages are ignored by default
|
|
882
|
-
}
|
|
883
|
-
}
|
|
884
|
-
|
|
885
|
-
const now = Date.now();
|
|
886
|
-
if (now - lastProgressAt > 200) {
|
|
887
|
-
lastProgressAt = now;
|
|
888
|
-
onProgress?.({
|
|
889
|
-
phase: 'session-progress',
|
|
890
|
-
sessionIndex,
|
|
891
|
-
messagesProcessed: result.totalMessages,
|
|
892
|
-
imported: result.importedPrompts + result.importedResponses,
|
|
893
|
-
skipped: result.skippedDuplicates
|
|
894
|
-
});
|
|
895
|
-
}
|
|
896
|
-
} catch (parseError) {
|
|
897
|
-
result.errors.push(`Parse error: ${parseError}`);
|
|
898
|
-
}
|
|
899
|
-
}
|
|
900
|
-
} finally {
|
|
901
|
-
await flushTextBuffer();
|
|
902
|
-
rl.close();
|
|
903
|
-
fileStream.close();
|
|
904
|
-
}
|
|
905
|
-
|
|
906
|
-
await this.memoryService.endSession(sessionId);
|
|
907
|
-
|
|
908
|
-
if (effectiveProjectPath) {
|
|
909
|
-
registerSession(sessionId, effectiveProjectPath);
|
|
910
|
-
}
|
|
911
|
-
|
|
912
|
-
if (options.verbose) {
|
|
913
|
-
console.log(`Imported ${result.importedPrompts} prompts, ${result.importedResponses} responses from ${filePath}`);
|
|
914
|
-
}
|
|
915
|
-
|
|
916
|
-
return result;
|
|
917
|
-
}
|
|
918
|
-
|
|
919
|
-
async listAvailableSessions(projectPath?: string): Promise<Array<{
|
|
920
|
-
sessionId: string;
|
|
921
|
-
filePath: string;
|
|
922
|
-
size: number;
|
|
923
|
-
modifiedAt: Date;
|
|
924
|
-
}>> {
|
|
925
|
-
const sessionsRoot = this.getSessionsRoot();
|
|
926
|
-
if (!fs.existsSync(sessionsRoot)) return [];
|
|
927
|
-
|
|
928
|
-
const files = this.listSessionFilesRecursive(sessionsRoot);
|
|
929
|
-
const sessions: Array<{ sessionId: string; filePath: string; size: number; modifiedAt: Date }> = [];
|
|
930
|
-
|
|
931
|
-
const normalizedTarget = projectPath ? normalizeMaybeRealpath(projectPath) : null;
|
|
932
|
-
|
|
933
|
-
for (const filePath of files) {
|
|
934
|
-
try {
|
|
935
|
-
const stats = fs.statSync(filePath);
|
|
936
|
-
|
|
937
|
-
let sessionId = this.deriveSessionIdFromFileName(filePath) ?? path.basename(filePath, '.jsonl');
|
|
938
|
-
if (normalizedTarget) {
|
|
939
|
-
const meta = await this.readSessionMeta(filePath);
|
|
940
|
-
if (!meta.cwd) continue;
|
|
941
|
-
if (normalizeMaybeRealpath(meta.cwd) !== normalizedTarget) continue;
|
|
942
|
-
sessionId = meta.sessionId ?? sessionId;
|
|
943
|
-
}
|
|
944
|
-
|
|
945
|
-
sessions.push({
|
|
946
|
-
sessionId,
|
|
947
|
-
filePath,
|
|
948
|
-
size: stats.size,
|
|
949
|
-
modifiedAt: stats.mtime
|
|
950
|
-
});
|
|
951
|
-
} catch {
|
|
952
|
-
// ignore
|
|
953
|
-
}
|
|
954
|
-
}
|
|
955
|
-
|
|
956
|
-
sessions.sort((a, b) => b.modifiedAt.getTime() - a.modifiedAt.getTime());
|
|
957
|
-
return sessions;
|
|
958
|
-
}
|
|
959
|
-
}
|
|
960
|
-
|
|
961
|
-
export function createCodexSessionHistoryImporter(
|
|
962
|
-
memoryService: MemoryService,
|
|
963
|
-
options: CodexSessionHistoryImporterOptions = {}
|
|
964
|
-
): CodexSessionHistoryImporter {
|
|
965
|
-
return new CodexSessionHistoryImporter(memoryService, options);
|
|
966
|
-
}
|