oh-my-claude-sisyphus 3.4.1 → 3.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (135) hide show
  1. package/commands/cancel-ultraqa.md +1 -1
  2. package/dist/__tests__/analytics/analytics-summary.test.d.ts +2 -0
  3. package/dist/__tests__/analytics/analytics-summary.test.d.ts.map +1 -0
  4. package/dist/__tests__/analytics/analytics-summary.test.js +267 -0
  5. package/dist/__tests__/analytics/analytics-summary.test.js.map +1 -0
  6. package/dist/__tests__/analytics/backfill-dedup.test.d.ts +2 -0
  7. package/dist/__tests__/analytics/backfill-dedup.test.d.ts.map +1 -0
  8. package/dist/__tests__/analytics/backfill-dedup.test.js +179 -0
  9. package/dist/__tests__/analytics/backfill-dedup.test.js.map +1 -0
  10. package/dist/__tests__/analytics/backfill-engine.test.d.ts +2 -0
  11. package/dist/__tests__/analytics/backfill-engine.test.d.ts.map +1 -0
  12. package/dist/__tests__/analytics/backfill-engine.test.js +362 -0
  13. package/dist/__tests__/analytics/backfill-engine.test.js.map +1 -0
  14. package/dist/__tests__/analytics/cost-estimator.test.d.ts +2 -0
  15. package/dist/__tests__/analytics/cost-estimator.test.d.ts.map +1 -0
  16. package/dist/__tests__/analytics/cost-estimator.test.js +212 -0
  17. package/dist/__tests__/analytics/cost-estimator.test.js.map +1 -0
  18. package/dist/__tests__/analytics/output-estimator.test.d.ts +2 -0
  19. package/dist/__tests__/analytics/output-estimator.test.d.ts.map +1 -0
  20. package/dist/__tests__/analytics/output-estimator.test.js +106 -0
  21. package/dist/__tests__/analytics/output-estimator.test.js.map +1 -0
  22. package/dist/__tests__/analytics/token-extractor.test.d.ts +2 -0
  23. package/dist/__tests__/analytics/token-extractor.test.d.ts.map +1 -0
  24. package/dist/__tests__/analytics/token-extractor.test.js +121 -0
  25. package/dist/__tests__/analytics/token-extractor.test.js.map +1 -0
  26. package/dist/__tests__/analytics/transcript-parser.test.d.ts +2 -0
  27. package/dist/__tests__/analytics/transcript-parser.test.d.ts.map +1 -0
  28. package/dist/__tests__/analytics/transcript-parser.test.js +285 -0
  29. package/dist/__tests__/analytics/transcript-parser.test.js.map +1 -0
  30. package/dist/__tests__/analytics/transcript-scanner.test.d.ts +2 -0
  31. package/dist/__tests__/analytics/transcript-scanner.test.d.ts.map +1 -0
  32. package/dist/__tests__/analytics/transcript-scanner.test.js +401 -0
  33. package/dist/__tests__/analytics/transcript-scanner.test.js.map +1 -0
  34. package/dist/__tests__/analytics/transcript-token-extractor.test.d.ts +2 -0
  35. package/dist/__tests__/analytics/transcript-token-extractor.test.d.ts.map +1 -0
  36. package/dist/__tests__/analytics/transcript-token-extractor.test.js +175 -0
  37. package/dist/__tests__/analytics/transcript-token-extractor.test.js.map +1 -0
  38. package/dist/__tests__/hud/auto-tracking.integration.test.d.ts +2 -0
  39. package/dist/__tests__/hud/auto-tracking.integration.test.d.ts.map +1 -0
  40. package/dist/__tests__/hud/auto-tracking.integration.test.js +12 -0
  41. package/dist/__tests__/hud/auto-tracking.integration.test.js.map +1 -0
  42. package/dist/analytics/analytics-summary.d.ts +47 -0
  43. package/dist/analytics/analytics-summary.d.ts.map +1 -0
  44. package/dist/analytics/analytics-summary.js +170 -0
  45. package/dist/analytics/analytics-summary.js.map +1 -0
  46. package/dist/analytics/backfill-dedup.d.ts +49 -0
  47. package/dist/analytics/backfill-dedup.d.ts.map +1 -0
  48. package/dist/analytics/backfill-dedup.js +118 -0
  49. package/dist/analytics/backfill-dedup.js.map +1 -0
  50. package/dist/analytics/backfill-engine.d.ts +59 -0
  51. package/dist/analytics/backfill-engine.d.ts.map +1 -0
  52. package/dist/analytics/backfill-engine.js +163 -0
  53. package/dist/analytics/backfill-engine.js.map +1 -0
  54. package/dist/analytics/index.d.ts +8 -0
  55. package/dist/analytics/index.d.ts.map +1 -1
  56. package/dist/analytics/index.js +10 -0
  57. package/dist/analytics/index.js.map +1 -1
  58. package/dist/analytics/output-estimator.d.ts +26 -0
  59. package/dist/analytics/output-estimator.d.ts.map +1 -0
  60. package/dist/analytics/output-estimator.js +61 -0
  61. package/dist/analytics/output-estimator.js.map +1 -0
  62. package/dist/analytics/token-extractor.d.ts +31 -0
  63. package/dist/analytics/token-extractor.d.ts.map +1 -0
  64. package/dist/analytics/token-extractor.js +57 -0
  65. package/dist/analytics/token-extractor.js.map +1 -0
  66. package/dist/analytics/token-tracker.d.ts +7 -1
  67. package/dist/analytics/token-tracker.d.ts.map +1 -1
  68. package/dist/analytics/token-tracker.js +81 -0
  69. package/dist/analytics/token-tracker.js.map +1 -1
  70. package/dist/analytics/transcript-parser.d.ts +42 -0
  71. package/dist/analytics/transcript-parser.d.ts.map +1 -0
  72. package/dist/analytics/transcript-parser.js +90 -0
  73. package/dist/analytics/transcript-parser.js.map +1 -0
  74. package/dist/analytics/transcript-scanner.d.ts +50 -0
  75. package/dist/analytics/transcript-scanner.d.ts.map +1 -0
  76. package/dist/analytics/transcript-scanner.js +149 -0
  77. package/dist/analytics/transcript-scanner.js.map +1 -0
  78. package/dist/analytics/transcript-token-extractor.d.ts +19 -0
  79. package/dist/analytics/transcript-token-extractor.d.ts.map +1 -0
  80. package/dist/analytics/transcript-token-extractor.js +89 -0
  81. package/dist/analytics/transcript-token-extractor.js.map +1 -0
  82. package/dist/analytics/types.d.ts +52 -0
  83. package/dist/analytics/types.d.ts.map +1 -1
  84. package/dist/analytics/types.js.map +1 -1
  85. package/dist/cli/analytics.js +26 -1
  86. package/dist/cli/analytics.js.map +1 -1
  87. package/dist/cli/commands/backfill.d.ts +15 -0
  88. package/dist/cli/commands/backfill.d.ts.map +1 -0
  89. package/dist/cli/commands/backfill.js +146 -0
  90. package/dist/cli/commands/backfill.js.map +1 -0
  91. package/dist/cli/commands/stats.d.ts +1 -0
  92. package/dist/cli/commands/stats.d.ts.map +1 -1
  93. package/dist/cli/commands/stats.js +67 -31
  94. package/dist/cli/commands/stats.js.map +1 -1
  95. package/dist/cli/components/CostDashboard.d.ts +15 -0
  96. package/dist/cli/components/CostDashboard.d.ts.map +1 -0
  97. package/dist/cli/components/CostDashboard.js +15 -0
  98. package/dist/cli/components/CostDashboard.js.map +1 -0
  99. package/dist/cli/components/LiveStats.d.ts +16 -0
  100. package/dist/cli/components/LiveStats.d.ts.map +1 -0
  101. package/dist/cli/components/LiveStats.js +16 -0
  102. package/dist/cli/components/LiveStats.js.map +1 -0
  103. package/dist/cli/components/SessionBrowser.d.ts +14 -0
  104. package/dist/cli/components/SessionBrowser.d.ts.map +1 -0
  105. package/dist/cli/components/SessionBrowser.js +14 -0
  106. package/dist/cli/components/SessionBrowser.js.map +1 -0
  107. package/dist/cli/index.js +159 -3
  108. package/dist/cli/index.js.map +1 -1
  109. package/dist/cli/tui.d.ts +21 -0
  110. package/dist/cli/tui.d.ts.map +1 -0
  111. package/dist/cli/tui.js +21 -0
  112. package/dist/cli/tui.js.map +1 -0
  113. package/dist/hud/analytics-display.d.ts +16 -0
  114. package/dist/hud/analytics-display.d.ts.map +1 -1
  115. package/dist/hud/analytics-display.js +42 -0
  116. package/dist/hud/analytics-display.js.map +1 -1
  117. package/dist/hud/index.js +90 -3
  118. package/dist/hud/index.js.map +1 -1
  119. package/dist/hud/render.d.ts.map +1 -1
  120. package/dist/hud/render.js +27 -1
  121. package/dist/hud/render.js.map +1 -1
  122. package/dist/hud/types.d.ts +2 -0
  123. package/dist/hud/types.d.ts.map +1 -1
  124. package/dist/hud/types.js.map +1 -1
  125. package/docs/ANALYTICS-SYSTEM.md +150 -0
  126. package/docs/FULL-README.md +2 -2
  127. package/hooks/keyword-detector.sh +1 -1
  128. package/package.json +1 -1
  129. package/scripts/keyword-detector.mjs +1 -1
  130. package/scripts/persistent-mode.mjs +1 -1
  131. package/scripts/test-mutual-exclusion.ts +4 -4
  132. package/scripts/test-remember-tags.ts +6 -6
  133. package/scripts/test-session-injection.ts +4 -4
  134. package/skills/cancel-ultraqa/SKILL.md +1 -1
  135. package/skills/omc-setup/SKILL.md +30 -5
@@ -0,0 +1,47 @@
1
+ /**
2
+ * Analytics Summary - Fast session analytics loading
3
+ *
4
+ * This module provides mtime-based caching for <10ms session load times.
5
+ */
6
+ export interface AnalyticsSummary {
7
+ sessionId: string;
8
+ lastUpdated: string;
9
+ lastLogOffset: number;
10
+ totals: {
11
+ inputTokens: number;
12
+ outputTokens: number;
13
+ cacheCreationTokens: number;
14
+ cacheReadTokens: number;
15
+ estimatedCost: number;
16
+ };
17
+ topAgents: Array<{
18
+ agent: string;
19
+ cost: number;
20
+ tokens: number;
21
+ }>;
22
+ cacheHitRate: number;
23
+ }
24
+ /**
25
+ * Get summary file path for session ID.
26
+ */
27
+ export declare function getSummaryPath(sessionId: string): string;
28
+ /**
29
+ * Initialize empty summary for new session.
30
+ */
31
+ export declare function createEmptySummary(sessionId: string): AnalyticsSummary;
32
+ /**
33
+ * Load analytics summary with mtime-based caching.
34
+ *
35
+ * Performance target: <10ms (vs 50-100ms full JSONL rebuild)
36
+ *
37
+ * Strategy:
38
+ * 1. Check if summary file exists and is fresh (mtime >= log mtime)
39
+ * 2. If fresh → return cached summary
40
+ * 3. If stale → rebuild incrementally from lastLogOffset
41
+ */
42
+ export declare function loadAnalyticsFast(sessionId: string): Promise<AnalyticsSummary | null>;
43
+ /**
44
+ * Force rebuild summary from scratch (ignore cache)
45
+ */
46
+ export declare function rebuildAnalyticsSummary(sessionId: string): Promise<AnalyticsSummary>;
47
+ //# sourceMappingURL=analytics-summary.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"analytics-summary.d.ts","sourceRoot":"","sources":["../../src/analytics/analytics-summary.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,MAAM,WAAW,gBAAgB;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,MAAM,EAAE;QACN,WAAW,EAAE,MAAM,CAAC;QACpB,YAAY,EAAE,MAAM,CAAC;QACrB,mBAAmB,EAAE,MAAM,CAAC;QAC5B,eAAe,EAAE,MAAM,CAAC;QACxB,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC;IACF,SAAS,EAAE,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAClE,YAAY,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAExD;AAED;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,SAAS,EAAE,MAAM,GAAG,gBAAgB,CAetE;AA4HD;;;;;;;;;GASG;AACH,wBAAsB,iBAAiB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAqB3F;AAED;;GAEG;AACH,wBAAsB,uBAAuB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAY1F"}
@@ -0,0 +1,170 @@
1
+ /**
2
+ * Analytics Summary - Fast session analytics loading
3
+ *
4
+ * This module provides mtime-based caching for <10ms session load times.
5
+ */
6
+ /**
7
+ * Get summary file path for session ID.
8
+ */
9
+ export function getSummaryPath(sessionId) {
10
+ return `.omc/state/analytics-summary-${sessionId}.json`;
11
+ }
12
+ /**
13
+ * Initialize empty summary for new session.
14
+ */
15
+ export function createEmptySummary(sessionId) {
16
+ return {
17
+ sessionId,
18
+ lastUpdated: new Date().toISOString(),
19
+ lastLogOffset: 0,
20
+ totals: {
21
+ inputTokens: 0,
22
+ outputTokens: 0,
23
+ cacheCreationTokens: 0,
24
+ cacheReadTokens: 0,
25
+ estimatedCost: 0
26
+ },
27
+ topAgents: [],
28
+ cacheHitRate: 0
29
+ };
30
+ }
31
+ import * as fs from 'fs/promises';
32
+ import * as path from 'path';
33
+ import { calculateCost } from './cost-estimator.js';
34
+ /**
35
+ * Get file modification time
36
+ */
37
+ async function getFileMtime(filePath) {
38
+ try {
39
+ const stats = await fs.stat(filePath);
40
+ return stats.mtime;
41
+ }
42
+ catch {
43
+ return null;
44
+ }
45
+ }
46
+ /**
47
+ * Update top agents list
48
+ */
49
+ function updateTopAgents(summary, agentName, cost, tokens) {
50
+ let agent = summary.topAgents.find(a => a.agent === agentName);
51
+ if (!agent) {
52
+ agent = { agent: agentName, cost: 0, tokens: 0 };
53
+ summary.topAgents.push(agent);
54
+ }
55
+ agent.cost += cost;
56
+ agent.tokens += tokens;
57
+ // Sort by cost and keep top 10
58
+ summary.topAgents.sort((a, b) => b.cost - a.cost);
59
+ summary.topAgents = summary.topAgents.slice(0, 10);
60
+ }
61
+ /**
62
+ * Calculate cache hit rate percentage
63
+ */
64
+ function calculateCacheHitRate(totals) {
65
+ const total = totals.inputTokens + totals.cacheCreationTokens;
66
+ if (total === 0)
67
+ return 0;
68
+ return (totals.cacheReadTokens / total) * 100;
69
+ }
70
+ /**
71
+ * Rebuild summary incrementally from JSONL log.
72
+ *
73
+ * Starts from lastLogOffset to avoid re-processing entire log.
74
+ */
75
+ async function rebuildSummaryIncremental(sessionId, summaryPath, logPath) {
76
+ let summary;
77
+ let startOffset = 0;
78
+ try {
79
+ // Try to load existing summary
80
+ const content = await fs.readFile(summaryPath, 'utf-8');
81
+ summary = JSON.parse(content);
82
+ startOffset = summary.lastLogOffset;
83
+ }
84
+ catch {
85
+ // Start fresh
86
+ summary = createEmptySummary(sessionId);
87
+ }
88
+ // Read JSONL from lastLogOffset
89
+ const logContent = await fs.readFile(logPath, 'utf-8');
90
+ const lines = logContent.split('\n').slice(startOffset);
91
+ for (const line of lines) {
92
+ if (!line.trim())
93
+ continue;
94
+ const record = JSON.parse(line);
95
+ if (record.sessionId !== sessionId)
96
+ continue;
97
+ // Update totals
98
+ summary.totals.inputTokens += record.inputTokens || 0;
99
+ summary.totals.outputTokens += record.outputTokens || 0;
100
+ summary.totals.cacheCreationTokens += record.cacheCreationTokens || 0;
101
+ summary.totals.cacheReadTokens += record.cacheReadTokens || 0;
102
+ // Update cost
103
+ const cost = calculateCost({
104
+ modelName: record.modelName,
105
+ inputTokens: record.inputTokens,
106
+ outputTokens: record.outputTokens,
107
+ cacheCreationTokens: record.cacheCreationTokens,
108
+ cacheReadTokens: record.cacheReadTokens
109
+ });
110
+ summary.totals.estimatedCost += cost.totalCost;
111
+ // Update top agents
112
+ if (record.agentName) {
113
+ updateTopAgents(summary, record.agentName, cost.totalCost, record.inputTokens + record.outputTokens);
114
+ }
115
+ }
116
+ // Update metadata
117
+ summary.lastUpdated = new Date().toISOString();
118
+ summary.lastLogOffset = startOffset + lines.length;
119
+ summary.cacheHitRate = calculateCacheHitRate(summary.totals);
120
+ // Save updated summary
121
+ await fs.mkdir(path.dirname(summaryPath), { recursive: true });
122
+ await fs.writeFile(summaryPath, JSON.stringify(summary, null, 2));
123
+ return summary;
124
+ }
125
+ /**
126
+ * Load analytics summary with mtime-based caching.
127
+ *
128
+ * Performance target: <10ms (vs 50-100ms full JSONL rebuild)
129
+ *
130
+ * Strategy:
131
+ * 1. Check if summary file exists and is fresh (mtime >= log mtime)
132
+ * 2. If fresh → return cached summary
133
+ * 3. If stale → rebuild incrementally from lastLogOffset
134
+ */
135
+ export async function loadAnalyticsFast(sessionId) {
136
+ const summaryPath = path.resolve(process.cwd(), getSummaryPath(sessionId));
137
+ const logPath = path.resolve(process.cwd(), '.omc/state/token-tracking.jsonl');
138
+ try {
139
+ // Check if summary exists and is fresh
140
+ const summaryMtime = await getFileMtime(summaryPath);
141
+ const logMtime = await getFileMtime(logPath);
142
+ if (summaryMtime && logMtime && summaryMtime >= logMtime) {
143
+ // Summary is up-to-date, use cached version
144
+ const content = await fs.readFile(summaryPath, 'utf-8');
145
+ return JSON.parse(content);
146
+ }
147
+ // Need to rebuild (incremental if summary exists)
148
+ return await rebuildSummaryIncremental(sessionId, summaryPath, logPath);
149
+ }
150
+ catch (error) {
151
+ // No summary exists yet or error reading
152
+ return null;
153
+ }
154
+ }
155
+ /**
156
+ * Force rebuild summary from scratch (ignore cache)
157
+ */
158
+ export async function rebuildAnalyticsSummary(sessionId) {
159
+ const summaryPath = path.resolve(process.cwd(), getSummaryPath(sessionId));
160
+ const logPath = path.resolve(process.cwd(), '.omc/state/token-tracking.jsonl');
161
+ // Delete existing summary to force full rebuild
162
+ try {
163
+ await fs.unlink(summaryPath);
164
+ }
165
+ catch {
166
+ // Ignore if doesn't exist
167
+ }
168
+ return await rebuildSummaryIncremental(sessionId, summaryPath, logPath);
169
+ }
170
+ //# sourceMappingURL=analytics-summary.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"analytics-summary.js","sourceRoot":"","sources":["../../src/analytics/analytics-summary.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAiBH;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,SAAiB;IAC9C,OAAO,gCAAgC,SAAS,OAAO,CAAC;AAC1D,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAAC,SAAiB;IAClD,OAAO;QACL,SAAS;QACT,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACrC,aAAa,EAAE,CAAC;QAChB,MAAM,EAAE;YACN,WAAW,EAAE,CAAC;YACd,YAAY,EAAE,CAAC;YACf,mBAAmB,EAAE,CAAC;YACtB,eAAe,EAAE,CAAC;YAClB,aAAa,EAAE,CAAC;SACjB;QACD,SAAS,EAAE,EAAE;QACb,YAAY,EAAE,CAAC;KAChB,CAAC;AACJ,CAAC;AAED,OAAO,KAAK,EAAE,MAAM,aAAa,CAAC;AAClC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAGpD;;GAEG;AACH,KAAK,UAAU,YAAY,CAAC,QAAgB;IAC1C,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACtC,OAAO,KAAK,CAAC,KAAK,CAAC;IACrB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,eAAe,CACtB,OAAyB,EACzB,SAAiB,EACjB,IAAY,EACZ,MAAc;IAEd,IAAI,KAAK,GAAG,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC;IAE/D,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,KAAK,GAAG,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;QACjD,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC;IAED,KAAK,CAAC,IAAI,IAAI,IAAI,CAAC;IACnB,KAAK,CAAC,MAAM,IAAI,MAAM,CAAC;IAEvB,+BAA+B;IAC/B,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;IAClD,OAAO,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AACrD,CAAC;AAED;;GAEG;AACH,SAAS,qBAAqB,CAAC,MAAkC;IAC/D,MAAM,KAAK,GAAG,MAAM,CAAC,WAAW,GAAG,MAAM,CAAC,mBAAmB,CAAC;IAC9D,IAAI,KAAK,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAC1B,OAAO,CAAC,MAAM,CAAC,eAAe,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC;AAChD,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,yBAAyB,CACtC,SAAiB,EACjB,WAAmB,EACnB,OAAe;IAEf,IAAI,OAAyB,CAAC;IAC9B,IAAI,WAAW,GAAG,CAAC,CAAC;IAEpB,IAAI,CAAC;QACH,+BAA+B;QAC/B,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;QACxD,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC9B,WAAW,GAAG,OAAO,CAAC,aAAa,CAAC;IACtC,CAAC;IAAC,MAAM,CAAC;QACP,cAAc;QACd,OAAO,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAC;IAC1C,CAAC;IAED,gCAAgC;IAChC,MAAM,UAAU,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IACvD,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;IAExD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;YAAE,SAAS;QAE3B,MAAM,MAAM,GAAe,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC5C,IAAI,MAAM,CAAC,SAAS,KAAK,SAAS;YAAE,SAAS;QAE7C,gBAAgB;QAChB,OAAO,CAAC,MAAM,CAAC,WAAW,IAAI,MAAM,CAAC,WAAW,IAAI,CAAC,CAAC;QACtD,OAAO,CAAC,MAAM,CAAC,YAAY,IAAI,MAAM,CAAC,YAAY,IAAI,CAAC,CAAC;QACxD,OAAO,CAAC,MAAM,CAAC,mBAAmB,IAAI,MAAM,CAAC,mBAAmB,IAAI,CAAC,CAAC;QACtE,OAAO,CAAC,MAAM,CAAC,eAAe,IAAI,MAAM,CAAC,eAAe,IAAI,CAAC,CAAC;QAE9D,cAAc;QACd,MAAM,IAAI,GAAG,aAAa,CAAC;YACzB,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,YAAY,EAAE,MAAM,CAAC,YAAY;YACjC,mBAAmB,EAAE,MAAM,CAAC,mBAAmB;YAC/C,eAAe,EAAE,MAAM,CAAC,eAAe;SACxC,CAAC,CAAC;QACH,OAAO,CAAC,MAAM,CAAC,aAAa,IAAI,IAAI,CAAC,SAAS,CAAC;QAE/C,oBAAoB;QACpB,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;YACrB,eAAe,CACb,OAAO,EACP,MAAM,CAAC,SAAS,EAChB,IAAI,CAAC,SAAS,EACd,MAAM,CAAC,WAAW,GAAG,MAAM,CAAC,YAAY,CACzC,CAAC;QACJ,CAAC;IACH,CAAC;IAED,kBAAkB;IAClB,OAAO,CAAC,WAAW,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC/C,OAAO,CAAC,aAAa,GAAG,WAAW,GAAG,KAAK,CAAC,MAAM,CAAC;IACnD,OAAO,CAAC,YAAY,GAAG,qBAAqB,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAE7D,uBAAuB;IACvB,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC/D,MAAM,EAAE,CAAC,SAAS,CAAC,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAElE,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,SAAiB;IACvD,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,cAAc,CAAC,SAAS,CAAC,CAAC,CAAC;IAC3E,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,iCAAiC,CAAC,CAAC;IAE/E,IAAI,CAAC;QACH,uCAAuC;QACvC,MAAM,YAAY,GAAG,MAAM,YAAY,CAAC,WAAW,CAAC,CAAC;QACrD,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,OAAO,CAAC,CAAC;QAE7C,IAAI,YAAY,IAAI,QAAQ,IAAI,YAAY,IAAI,QAAQ,EAAE,CAAC;YACzD,4CAA4C;YAC5C,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;YACxD,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC7B,CAAC;QAED,kDAAkD;QAClD,OAAO,MAAM,yBAAyB,CAAC,SAAS,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC;IAC1E,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,yCAAyC;QACzC,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,SAAiB;IAC7D,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,cAAc,CAAC,SAAS,CAAC,CAAC,CAAC;IAC3E,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,iCAAiC,CAAC,CAAC;IAE/E,gDAAgD;IAChD,IAAI,CAAC;QACH,MAAM,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;IAC/B,CAAC;IAAC,MAAM,CAAC;QACP,0BAA0B;IAC5B,CAAC;IAED,OAAO,MAAM,yBAAyB,CAAC,SAAS,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC;AAC1E,CAAC"}
@@ -0,0 +1,49 @@
1
+ export interface DedupIndex {
2
+ processedIds: string[];
3
+ lastBackfillTime: string;
4
+ totalProcessed: number;
5
+ }
6
+ /**
7
+ * BackfillDedup provides fast deduplication for backfill operations.
8
+ *
9
+ * Uses in-memory Set for O(1) lookups and persists state to disk.
10
+ * Loads existing entries from token-tracking.jsonl on initialization.
11
+ */
12
+ export declare class BackfillDedup {
13
+ private processedSet;
14
+ private totalProcessed;
15
+ private lastBackfillTime;
16
+ /**
17
+ * Load existing processed IDs from backfill-index.json and scan token-tracking.jsonl
18
+ */
19
+ load(): Promise<void>;
20
+ /**
21
+ * Generate unique ID for a token usage entry
22
+ * Uses SHA256 hash to match transcript-token-extractor.ts format
23
+ */
24
+ private generateEntryId;
25
+ /**
26
+ * Check if an entry ID has already been processed
27
+ */
28
+ isProcessed(entryId: string): boolean;
29
+ /**
30
+ * Mark an entry ID as processed
31
+ */
32
+ markProcessed(entryId: string): void;
33
+ /**
34
+ * Persist deduplication state to backfill-index.json
35
+ */
36
+ save(): Promise<void>;
37
+ /**
38
+ * Clear all processed entries and delete index file
39
+ */
40
+ reset(): Promise<void>;
41
+ /**
42
+ * Get current statistics
43
+ */
44
+ getStats(): {
45
+ totalProcessed: number;
46
+ lastBackfillTime: string;
47
+ };
48
+ }
49
+ //# sourceMappingURL=backfill-dedup.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"backfill-dedup.d.ts","sourceRoot":"","sources":["../../src/analytics/backfill-dedup.ts"],"names":[],"mappings":"AAQA,MAAM,WAAW,UAAU;IACzB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,gBAAgB,EAAE,MAAM,CAAC;IACzB,cAAc,EAAE,MAAM,CAAC;CACxB;AAED;;;;;GAKG;AACH,qBAAa,aAAa;IACxB,OAAO,CAAC,YAAY,CAA0B;IAC9C,OAAO,CAAC,cAAc,CAAa;IACnC,OAAO,CAAC,gBAAgB,CAAoC;IAE5D;;OAEG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAuC3B;;;OAGG;IACH,OAAO,CAAC,eAAe;IAMvB;;OAEG;IACH,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO;IAIrC;;OAEG;IACH,aAAa,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI;IAOpC;;OAEG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAe3B;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAc5B;;OAEG;IACH,QAAQ,IAAI;QAAE,cAAc,EAAE,MAAM,CAAC;QAAC,gBAAgB,EAAE,MAAM,CAAA;KAAE;CAMjE"}
@@ -0,0 +1,118 @@
1
+ import { createHash } from 'crypto';
2
+ import * as fs from 'fs/promises';
3
+ import * as path from 'path';
4
+ const TOKEN_LOG_FILE = '.omc/state/token-tracking.jsonl';
5
+ const DEDUP_INDEX_FILE = '.omc/state/backfill-index.json';
6
+ /**
7
+ * BackfillDedup provides fast deduplication for backfill operations.
8
+ *
9
+ * Uses in-memory Set for O(1) lookups and persists state to disk.
10
+ * Loads existing entries from token-tracking.jsonl on initialization.
11
+ */
12
+ export class BackfillDedup {
13
+ processedSet = new Set();
14
+ totalProcessed = 0;
15
+ lastBackfillTime = new Date().toISOString();
16
+ /**
17
+ * Load existing processed IDs from backfill-index.json and scan token-tracking.jsonl
18
+ */
19
+ async load() {
20
+ const indexPath = path.resolve(process.cwd(), DEDUP_INDEX_FILE);
21
+ const logPath = path.resolve(process.cwd(), TOKEN_LOG_FILE);
22
+ // Load persisted index first
23
+ try {
24
+ const indexContent = await fs.readFile(indexPath, 'utf-8');
25
+ const index = JSON.parse(indexContent);
26
+ this.processedSet = new Set(index.processedIds);
27
+ this.totalProcessed = index.totalProcessed;
28
+ this.lastBackfillTime = index.lastBackfillTime;
29
+ }
30
+ catch (error) {
31
+ // Index doesn't exist yet, will be created on save
32
+ }
33
+ // Scan token-tracking.jsonl to ensure all existing entries are marked
34
+ try {
35
+ const logContent = await fs.readFile(logPath, 'utf-8');
36
+ const lines = logContent.trim().split('\n').filter(line => line.length > 0);
37
+ for (const line of lines) {
38
+ try {
39
+ const record = JSON.parse(line);
40
+ const entryId = this.generateEntryId(record);
41
+ if (!this.processedSet.has(entryId)) {
42
+ this.processedSet.add(entryId);
43
+ this.totalProcessed++;
44
+ }
45
+ }
46
+ catch (parseError) {
47
+ // Skip malformed lines
48
+ }
49
+ }
50
+ }
51
+ catch (error) {
52
+ // Log file doesn't exist yet, which is fine
53
+ }
54
+ }
55
+ /**
56
+ * Generate unique ID for a token usage entry
57
+ * Uses SHA256 hash to match transcript-token-extractor.ts format
58
+ */
59
+ generateEntryId(record) {
60
+ const hash = createHash('sha256');
61
+ hash.update(`${record.sessionId}:${record.timestamp}:${record.modelName}`);
62
+ return hash.digest('hex');
63
+ }
64
+ /**
65
+ * Check if an entry ID has already been processed
66
+ */
67
+ isProcessed(entryId) {
68
+ return this.processedSet.has(entryId);
69
+ }
70
+ /**
71
+ * Mark an entry ID as processed
72
+ */
73
+ markProcessed(entryId) {
74
+ if (!this.processedSet.has(entryId)) {
75
+ this.processedSet.add(entryId);
76
+ this.totalProcessed++;
77
+ }
78
+ }
79
+ /**
80
+ * Persist deduplication state to backfill-index.json
81
+ */
82
+ async save() {
83
+ const indexPath = path.resolve(process.cwd(), DEDUP_INDEX_FILE);
84
+ const indexDir = path.dirname(indexPath);
85
+ await fs.mkdir(indexDir, { recursive: true });
86
+ const index = {
87
+ processedIds: Array.from(this.processedSet),
88
+ lastBackfillTime: new Date().toISOString(),
89
+ totalProcessed: this.totalProcessed
90
+ };
91
+ await fs.writeFile(indexPath, JSON.stringify(index, null, 2), 'utf-8');
92
+ }
93
+ /**
94
+ * Clear all processed entries and delete index file
95
+ */
96
+ async reset() {
97
+ this.processedSet.clear();
98
+ this.totalProcessed = 0;
99
+ this.lastBackfillTime = new Date().toISOString();
100
+ const indexPath = path.resolve(process.cwd(), DEDUP_INDEX_FILE);
101
+ try {
102
+ await fs.unlink(indexPath);
103
+ }
104
+ catch (error) {
105
+ // File might not exist, which is fine
106
+ }
107
+ }
108
+ /**
109
+ * Get current statistics
110
+ */
111
+ getStats() {
112
+ return {
113
+ totalProcessed: this.totalProcessed,
114
+ lastBackfillTime: this.lastBackfillTime
115
+ };
116
+ }
117
+ }
118
+ //# sourceMappingURL=backfill-dedup.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"backfill-dedup.js","sourceRoot":"","sources":["../../src/analytics/backfill-dedup.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,KAAK,EAAE,MAAM,aAAa,CAAC;AAClC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAG7B,MAAM,cAAc,GAAG,iCAAiC,CAAC;AACzD,MAAM,gBAAgB,GAAG,gCAAgC,CAAC;AAQ1D;;;;;GAKG;AACH,MAAM,OAAO,aAAa;IAChB,YAAY,GAAgB,IAAI,GAAG,EAAE,CAAC;IACtC,cAAc,GAAW,CAAC,CAAC;IAC3B,gBAAgB,GAAW,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAE5D;;OAEG;IACH,KAAK,CAAC,IAAI;QACR,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,gBAAgB,CAAC,CAAC;QAChE,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,cAAc,CAAC,CAAC;QAE5D,6BAA6B;QAC7B,IAAI,CAAC;YACH,MAAM,YAAY,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;YAC3D,MAAM,KAAK,GAAe,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;YAEnD,IAAI,CAAC,YAAY,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;YAChD,IAAI,CAAC,cAAc,GAAG,KAAK,CAAC,cAAc,CAAC;YAC3C,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC,gBAAgB,CAAC;QACjD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,mDAAmD;QACrD,CAAC;QAED,sEAAsE;QACtE,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YACvD,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAE5E,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,IAAI,CAAC;oBACH,MAAM,MAAM,GAAe,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;oBAC5C,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;oBAE7C,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;wBACpC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;wBAC/B,IAAI,CAAC,cAAc,EAAE,CAAC;oBACxB,CAAC;gBACH,CAAC;gBAAC,OAAO,UAAU,EAAE,CAAC;oBACpB,uBAAuB;gBACzB,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,4CAA4C;QAC9C,CAAC;IACH,CAAC;IAED;;;OAGG;IACK,eAAe,CAAC,MAAkB;QACxC,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;QAClC,IAAI,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC;QAC3E,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC5B,CAAC;IAED;;OAEG;IACH,WAAW,CAAC,OAAe;QACzB,OAAO,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IACxC,CAAC;IAED;;OAEG;IACH,aAAa,CAAC,OAAe;QAC3B,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YACpC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YAC/B,IAAI,CAAC,cAAc,EAAE,CAAC;QACxB,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAI;QACR,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,gBAAgB,CAAC,CAAC;QAChE,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAEzC,MAAM,EAAE,CAAC,KAAK,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAE9C,MAAM,KAAK,GAAe;YACxB,YAAY,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC;YAC3C,gBAAgB,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YAC1C,cAAc,EAAE,IAAI,CAAC,cAAc;SACpC,CAAC;QAEF,MAAM,EAAE,CAAC,SAAS,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IACzE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;QAC1B,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC;QACxB,IAAI,CAAC,gBAAgB,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAEjD,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,gBAAgB,CAAC,CAAC;QAEhE,IAAI,CAAC;YACH,MAAM,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAC7B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,sCAAsC;QACxC,CAAC;IACH,CAAC;IAED;;OAEG;IACH,QAAQ;QACN,OAAO;YACL,cAAc,EAAE,IAAI,CAAC,cAAc;YACnC,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;SACxC,CAAC;IACJ,CAAC;CACF"}
@@ -0,0 +1,59 @@
1
+ import { EventEmitter } from 'events';
2
+ export interface BackfillOptions {
3
+ projectFilter?: string;
4
+ dateFrom?: Date;
5
+ dateTo?: Date;
6
+ dryRun?: boolean;
7
+ verbose?: boolean;
8
+ }
9
+ export interface BackfillResult {
10
+ filesProcessed: number;
11
+ entriesAdded: number;
12
+ duplicatesSkipped: number;
13
+ errorsEncountered: number;
14
+ totalCostDiscovered: number;
15
+ timeElapsed: number;
16
+ }
17
+ export interface BackfillProgress {
18
+ currentFile: string;
19
+ filesProcessed: number;
20
+ totalFiles: number;
21
+ entriesAdded: number;
22
+ duplicatesSkipped: number;
23
+ currentCost: number;
24
+ }
25
+ /**
26
+ * BackfillEngine orchestrates the offline transcript analysis pipeline.
27
+ *
28
+ * Pipeline:
29
+ * 1. Scan for transcripts
30
+ * 2. Parse each transcript file (streaming)
31
+ * 3. Extract token usage from entries
32
+ * 4. Deduplicate
33
+ * 5. Write to token-tracking.jsonl (batch)
34
+ *
35
+ * Emits 'progress' events during execution.
36
+ */
37
+ export declare class BackfillEngine extends EventEmitter {
38
+ private aborted;
39
+ private dedup;
40
+ private tracker;
41
+ constructor();
42
+ /**
43
+ * Abort the backfill operation
44
+ */
45
+ abort(): void;
46
+ /**
47
+ * Run the backfill process
48
+ */
49
+ run(options?: BackfillOptions): Promise<BackfillResult>;
50
+ /**
51
+ * Process a single transcript file
52
+ */
53
+ private processTranscript;
54
+ /**
55
+ * Write a batch of token usage entries to the tracker
56
+ */
57
+ private flushBatch;
58
+ }
59
+ //# sourceMappingURL=backfill-engine.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"backfill-engine.d.ts","sourceRoot":"","sources":["../../src/analytics/backfill-engine.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AAUtC,MAAM,WAAW,eAAe;IAC9B,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,EAAE,IAAI,CAAC;IAChB,MAAM,CAAC,EAAE,IAAI,CAAC;IACd,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,WAAW,cAAc;IAC7B,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,gBAAgB;IAC/B,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;;;;;;;;;;GAWG;AACH,qBAAa,cAAe,SAAQ,YAAY;IAC9C,OAAO,CAAC,OAAO,CAAkB;IACjC,OAAO,CAAC,KAAK,CAAgB;IAC7B,OAAO,CAAC,OAAO,CAAe;;IAQ9B;;OAEG;IACH,KAAK,IAAI,IAAI;IAIb;;OAEG;IACG,GAAG,CAAC,OAAO,GAAE,eAAoB,GAAG,OAAO,CAAC,cAAc,CAAC;IAwDjE;;OAEG;YACW,iBAAiB;IAyE/B;;OAEG;YACW,UAAU;CAczB"}
@@ -0,0 +1,163 @@
1
+ import { EventEmitter } from 'events';
2
+ import * as path from 'path';
3
+ import { scanTranscripts } from './transcript-scanner.js';
4
+ import { parseTranscript } from './transcript-parser.js';
5
+ import { BackfillDedup } from './backfill-dedup.js';
6
+ import { getTokenTracker } from './token-tracker.js';
7
+ import { calculateCost } from './cost-estimator.js';
8
+ import { extractTokenUsage } from './transcript-token-extractor.js';
9
+ /**
10
+ * BackfillEngine orchestrates the offline transcript analysis pipeline.
11
+ *
12
+ * Pipeline:
13
+ * 1. Scan for transcripts
14
+ * 2. Parse each transcript file (streaming)
15
+ * 3. Extract token usage from entries
16
+ * 4. Deduplicate
17
+ * 5. Write to token-tracking.jsonl (batch)
18
+ *
19
+ * Emits 'progress' events during execution.
20
+ */
21
+ export class BackfillEngine extends EventEmitter {
22
+ aborted = false;
23
+ dedup;
24
+ tracker;
25
+ constructor() {
26
+ super();
27
+ this.dedup = new BackfillDedup();
28
+ this.tracker = getTokenTracker();
29
+ }
30
+ /**
31
+ * Abort the backfill operation
32
+ */
33
+ abort() {
34
+ this.aborted = true;
35
+ }
36
+ /**
37
+ * Run the backfill process
38
+ */
39
+ async run(options = {}) {
40
+ const startTime = Date.now();
41
+ this.aborted = false;
42
+ // Load deduplication state
43
+ await this.dedup.load();
44
+ // Scan for transcripts
45
+ const scanResult = await scanTranscripts({
46
+ projectFilter: options.projectFilter,
47
+ minDate: options.dateFrom,
48
+ });
49
+ const result = {
50
+ filesProcessed: 0,
51
+ entriesAdded: 0,
52
+ duplicatesSkipped: 0,
53
+ errorsEncountered: 0,
54
+ totalCostDiscovered: 0,
55
+ timeElapsed: 0,
56
+ };
57
+ const totalFiles = scanResult.transcripts.length;
58
+ // Process each transcript file
59
+ for (const transcript of scanResult.transcripts) {
60
+ if (this.aborted) {
61
+ break;
62
+ }
63
+ // Filter by date range if specified
64
+ if (options.dateTo && transcript.modifiedTime > options.dateTo) {
65
+ continue;
66
+ }
67
+ try {
68
+ await this.processTranscript(transcript, options, result, totalFiles);
69
+ }
70
+ catch (error) {
71
+ result.errorsEncountered++;
72
+ if (options.verbose) {
73
+ console.error(`Error processing ${transcript.filePath}:`, error);
74
+ }
75
+ }
76
+ result.filesProcessed++;
77
+ }
78
+ // Save deduplication state
79
+ if (!options.dryRun) {
80
+ await this.dedup.save();
81
+ }
82
+ result.timeElapsed = Date.now() - startTime;
83
+ return result;
84
+ }
85
+ /**
86
+ * Process a single transcript file
87
+ */
88
+ async processTranscript(transcript, options, result, totalFiles) {
89
+ const batch = [];
90
+ const BATCH_SIZE = 100;
91
+ // Emit progress
92
+ this.emit('progress', {
93
+ currentFile: path.basename(transcript.filePath),
94
+ filesProcessed: result.filesProcessed,
95
+ totalFiles,
96
+ entriesAdded: result.entriesAdded,
97
+ duplicatesSkipped: result.duplicatesSkipped,
98
+ currentCost: result.totalCostDiscovered,
99
+ });
100
+ // Parse transcript (streaming)
101
+ for await (const entry of parseTranscript(transcript.filePath, {
102
+ onParseError: (line, error) => {
103
+ result.errorsEncountered++;
104
+ if (options.verbose) {
105
+ console.warn(`Parse error in ${transcript.filePath}: ${error.message}`);
106
+ }
107
+ },
108
+ })) {
109
+ if (this.aborted) {
110
+ break;
111
+ }
112
+ // Extract token usage
113
+ const extracted = extractTokenUsage(entry, transcript.sessionId, transcript.filePath);
114
+ if (!extracted) {
115
+ continue; // Skip entries without usage data
116
+ }
117
+ // Check deduplication
118
+ if (this.dedup.isProcessed(extracted.entryId)) {
119
+ result.duplicatesSkipped++;
120
+ continue;
121
+ }
122
+ // Calculate cost
123
+ const cost = calculateCost({
124
+ modelName: extracted.usage.modelName,
125
+ inputTokens: extracted.usage.inputTokens,
126
+ outputTokens: extracted.usage.outputTokens,
127
+ cacheCreationTokens: extracted.usage.cacheCreationTokens,
128
+ cacheReadTokens: extracted.usage.cacheReadTokens,
129
+ });
130
+ result.totalCostDiscovered += cost.totalCost;
131
+ // Add to batch
132
+ batch.push(extracted.usage);
133
+ this.dedup.markProcessed(extracted.entryId);
134
+ result.entriesAdded++;
135
+ // Flush batch if full
136
+ if (batch.length >= BATCH_SIZE && !options.dryRun) {
137
+ await this.flushBatch(batch);
138
+ }
139
+ }
140
+ // Flush remaining entries
141
+ if (batch.length > 0 && !options.dryRun) {
142
+ await this.flushBatch(batch);
143
+ }
144
+ }
145
+ /**
146
+ * Write a batch of token usage entries to the tracker
147
+ */
148
+ async flushBatch(batch) {
149
+ for (const usage of batch) {
150
+ await this.tracker.recordTokenUsage({
151
+ modelName: usage.modelName,
152
+ inputTokens: usage.inputTokens,
153
+ outputTokens: usage.outputTokens,
154
+ cacheCreationTokens: usage.cacheCreationTokens,
155
+ cacheReadTokens: usage.cacheReadTokens,
156
+ agentName: usage.agentName,
157
+ isEstimated: usage.isEstimated,
158
+ });
159
+ }
160
+ batch.length = 0; // Clear batch
161
+ }
162
+ }
163
+ //# sourceMappingURL=backfill-engine.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"backfill-engine.js","sourceRoot":"","sources":["../../src/analytics/backfill-engine.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AACtC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,eAAe,EAAkB,MAAM,yBAAyB,CAAC;AAC1E,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAEzD,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAgB,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACnE,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAC;AA4BpE;;;;;;;;;;;GAWG;AACH,MAAM,OAAO,cAAe,SAAQ,YAAY;IACtC,OAAO,GAAY,KAAK,CAAC;IACzB,KAAK,CAAgB;IACrB,OAAO,CAAe;IAE9B;QACE,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,KAAK,GAAG,IAAI,aAAa,EAAE,CAAC;QACjC,IAAI,CAAC,OAAO,GAAG,eAAe,EAAE,CAAC;IACnC,CAAC;IAED;;OAEG;IACH,KAAK;QACH,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;IACtB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,GAAG,CAAC,UAA2B,EAAE;QACrC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;QAErB,2BAA2B;QAC3B,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QAExB,uBAAuB;QACvB,MAAM,UAAU,GAAG,MAAM,eAAe,CAAC;YACvC,aAAa,EAAE,OAAO,CAAC,aAAa;YACpC,OAAO,EAAE,OAAO,CAAC,QAAQ;SAC1B,CAAC,CAAC;QAEH,MAAM,MAAM,GAAmB;YAC7B,cAAc,EAAE,CAAC;YACjB,YAAY,EAAE,CAAC;YACf,iBAAiB,EAAE,CAAC;YACpB,iBAAiB,EAAE,CAAC;YACpB,mBAAmB,EAAE,CAAC;YACtB,WAAW,EAAE,CAAC;SACf,CAAC;QAEF,MAAM,UAAU,GAAG,UAAU,CAAC,WAAW,CAAC,MAAM,CAAC;QAEjD,+BAA+B;QAC/B,KAAK,MAAM,UAAU,IAAI,UAAU,CAAC,WAAW,EAAE,CAAC;YAChD,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;gBACjB,MAAM;YACR,CAAC;YAED,oCAAoC;YACpC,IAAI,OAAO,CAAC,MAAM,IAAI,UAAU,CAAC,YAAY,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;gBAC/D,SAAS;YACX,CAAC;YAED,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,iBAAiB,CAAC,UAAU,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;YACxE,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,iBAAiB,EAAE,CAAC;gBAC3B,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;oBACpB,OAAO,CAAC,KAAK,CAAC,oBAAoB,UAAU,CAAC,QAAQ,GAAG,EAAE,KAAK,CAAC,CAAC;gBACnE,CAAC;YACH,CAAC;YAED,MAAM,CAAC,cAAc,EAAE,CAAC;QAC1B,CAAC;QAED,2BAA2B;QAC3B,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;YACpB,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QAC1B,CAAC;QAED,MAAM,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;QAC5C,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,iBAAiB,CAC7B,UAA0B,EAC1B,OAAwB,EACxB,MAAsB,EACtB,UAAkB;QAElB,MAAM,KAAK,GAAiB,EAAE,CAAC;QAC/B,MAAM,UAAU,GAAG,GAAG,CAAC;QAEvB,gBAAgB;QAChB,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE;YACpB,WAAW,EAAE,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,QAAQ,CAAC;YAC/C,cAAc,EAAE,MAAM,CAAC,cAAc;YACrC,UAAU;YACV,YAAY,EAAE,MAAM,CAAC,YAAY;YACjC,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;YAC3C,WAAW,EAAE,MAAM,CAAC,mBAAmB;SACpB,CAAC,CAAC;QAEvB,+BAA+B;QAC/B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,eAAe,CAAC,UAAU,CAAC,QAAQ,EAAE;YAC7D,YAAY,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;gBAC5B,MAAM,CAAC,iBAAiB,EAAE,CAAC;gBAC3B,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;oBACpB,OAAO,CAAC,IAAI,CAAC,kBAAkB,UAAU,CAAC,QAAQ,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;gBAC1E,CAAC;YACH,CAAC;SACF,CAAC,EAAE,CAAC;YACH,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;gBACjB,MAAM;YACR,CAAC;YAED,sBAAsB;YACtB,MAAM,SAAS,GAAG,iBAAiB,CAAC,KAAK,EAAE,UAAU,CAAC,SAAS,EAAE,UAAU,CAAC,QAAQ,CAAC,CAAC;YAEtF,IAAI,CAAC,SAAS,EAAE,CAAC;gBACf,SAAS,CAAC,kCAAkC;YAC9C,CAAC;YAED,sBAAsB;YACtB,IAAI,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC9C,MAAM,CAAC,iBAAiB,EAAE,CAAC;gBAC3B,SAAS;YACX,CAAC;YAED,iBAAiB;YACjB,MAAM,IAAI,GAAG,aAAa,CAAC;gBACzB,SAAS,EAAE,SAAS,CAAC,KAAK,CAAC,SAAS;gBACpC,WAAW,EAAE,SAAS,CAAC,KAAK,CAAC,WAAW;gBACxC,YAAY,EAAE,SAAS,CAAC,KAAK,CAAC,YAAY;gBAC1C,mBAAmB,EAAE,SAAS,CAAC,KAAK,CAAC,mBAAmB;gBACxD,eAAe,EAAE,SAAS,CAAC,KAAK,CAAC,eAAe;aACjD,CAAC,CAAC;YAEH,MAAM,CAAC,mBAAmB,IAAI,IAAI,CAAC,SAAS,CAAC;YAE7C,eAAe;YACf,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;YAC5B,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;YAC5C,MAAM,CAAC,YAAY,EAAE,CAAC;YAEtB,sBAAsB;YACtB,IAAI,KAAK,CAAC,MAAM,IAAI,UAAU,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;gBAClD,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;YAC/B,CAAC;QACH,CAAC;QAED,0BAA0B;QAC1B,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;YACxC,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,UAAU,CAAC,KAAmB;QAC1C,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;YAC1B,MAAM,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC;gBAClC,SAAS,EAAE,KAAK,CAAC,SAAS;gBAC1B,WAAW,EAAE,KAAK,CAAC,WAAW;gBAC9B,YAAY,EAAE,KAAK,CAAC,YAAY;gBAChC,mBAAmB,EAAE,KAAK,CAAC,mBAAmB;gBAC9C,eAAe,EAAE,KAAK,CAAC,eAAe;gBACtC,SAAS,EAAE,KAAK,CAAC,SAAS;gBAC1B,WAAW,EAAE,KAAK,CAAC,WAAW;aAC/B,CAAC,CAAC;QACL,CAAC;QACD,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,cAAc;IAClC,CAAC;CACF"}
@@ -11,4 +11,12 @@ export * from './session-types.js';
11
11
  export * from './metrics-collector.js';
12
12
  export * from './query-engine.js';
13
13
  export * from './export.js';
14
+ export * from './token-extractor.js';
15
+ export * from './output-estimator.js';
16
+ export * from './analytics-summary.js';
17
+ export * from './transcript-scanner.js';
18
+ export * from './transcript-parser.js';
19
+ export * from './transcript-token-extractor.js';
20
+ export * from './backfill-dedup.js';
21
+ export * from './backfill-engine.js';
14
22
  //# sourceMappingURL=index.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/analytics/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,YAAY,CAAC;AAC3B,cAAc,oBAAoB,CAAC;AACnC,cAAc,qBAAqB,CAAC;AACpC,cAAc,sBAAsB,CAAC;AACrC,cAAc,oBAAoB,CAAC;AACnC,cAAc,wBAAwB,CAAC;AACvC,cAAc,mBAAmB,CAAC;AAClC,cAAc,aAAa,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/analytics/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,YAAY,CAAC;AAC3B,cAAc,oBAAoB,CAAC;AACnC,cAAc,qBAAqB,CAAC;AACpC,cAAc,sBAAsB,CAAC;AACrC,cAAc,oBAAoB,CAAC;AACnC,cAAc,wBAAwB,CAAC;AACvC,cAAc,mBAAmB,CAAC;AAClC,cAAc,aAAa,CAAC;AAG5B,cAAc,sBAAsB,CAAC;AACrC,cAAc,uBAAuB,CAAC;AACtC,cAAc,wBAAwB,CAAC;AAGvC,cAAc,yBAAyB,CAAC;AACxC,cAAc,wBAAwB,CAAC;AACvC,cAAc,iCAAiC,CAAC;AAChD,cAAc,qBAAqB,CAAC;AACpC,cAAc,sBAAsB,CAAC"}