@nocoo/pew 0.4.0 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +117 -8
- package/dist/cli.js.map +1 -1
- package/dist/commands/session-sync.d.ts +62 -0
- package/dist/commands/session-sync.d.ts.map +1 -0
- package/dist/commands/session-sync.js +443 -0
- package/dist/commands/session-sync.js.map +1 -0
- package/dist/commands/session-upload.d.ts +39 -0
- package/dist/commands/session-upload.d.ts.map +1 -0
- package/dist/commands/session-upload.js +45 -0
- package/dist/commands/session-upload.js.map +1 -0
- package/dist/commands/status.d.ts +9 -0
- package/dist/commands/status.d.ts.map +1 -1
- package/dist/commands/status.js +22 -11
- package/dist/commands/status.js.map +1 -1
- package/dist/commands/sync.d.ts +12 -0
- package/dist/commands/sync.d.ts.map +1 -1
- package/dist/commands/sync.js +159 -3
- package/dist/commands/sync.js.map +1 -1
- package/dist/commands/upload-engine.d.ts +63 -0
- package/dist/commands/upload-engine.d.ts.map +1 -0
- package/dist/commands/upload-engine.js +164 -0
- package/dist/commands/upload-engine.js.map +1 -0
- package/dist/commands/upload.d.ts +5 -21
- package/dist/commands/upload.d.ts.map +1 -1
- package/dist/commands/upload.js +11 -144
- package/dist/commands/upload.js.map +1 -1
- package/dist/discovery/sources.d.ts +5 -0
- package/dist/discovery/sources.d.ts.map +1 -1
- package/dist/discovery/sources.js +7 -0
- package/dist/discovery/sources.js.map +1 -1
- package/dist/parsers/claude-session.d.ts +19 -0
- package/dist/parsers/claude-session.d.ts.map +1 -0
- package/dist/parsers/claude-session.js +131 -0
- package/dist/parsers/claude-session.js.map +1 -0
- package/dist/parsers/codex-session.d.ts +24 -0
- package/dist/parsers/codex-session.d.ts.map +1 -0
- package/dist/parsers/codex-session.js +140 -0
- package/dist/parsers/codex-session.js.map +1 -0
- package/dist/parsers/codex.d.ts +37 -0
- package/dist/parsers/codex.d.ts.map +1 -0
- package/dist/parsers/codex.js +136 -0
- package/dist/parsers/codex.js.map +1 -0
- package/dist/parsers/gemini-session.d.ts +19 -0
- package/dist/parsers/gemini-session.d.ts.map +1 -0
- package/dist/parsers/gemini-session.js +103 -0
- package/dist/parsers/gemini-session.js.map +1 -0
- package/dist/parsers/openclaw-session.d.ts +20 -0
- package/dist/parsers/openclaw-session.d.ts.map +1 -0
- package/dist/parsers/openclaw-session.js +122 -0
- package/dist/parsers/openclaw-session.js.map +1 -0
- package/dist/parsers/opencode-session.d.ts +15 -0
- package/dist/parsers/opencode-session.d.ts.map +1 -0
- package/dist/parsers/opencode-session.js +131 -0
- package/dist/parsers/opencode-session.js.map +1 -0
- package/dist/parsers/opencode-sqlite-db.d.ts +29 -0
- package/dist/parsers/opencode-sqlite-db.d.ts.map +1 -0
- package/dist/parsers/opencode-sqlite-db.js +71 -0
- package/dist/parsers/opencode-sqlite-db.js.map +1 -0
- package/dist/parsers/opencode-sqlite-session.d.ts +32 -0
- package/dist/parsers/opencode-sqlite-session.d.ts.map +1 -0
- package/dist/parsers/opencode-sqlite-session.js +121 -0
- package/dist/parsers/opencode-sqlite-session.js.map +1 -0
- package/dist/parsers/opencode-sqlite.d.ts +53 -0
- package/dist/parsers/opencode-sqlite.d.ts.map +1 -0
- package/dist/parsers/opencode-sqlite.js +104 -0
- package/dist/parsers/opencode-sqlite.js.map +1 -0
- package/dist/storage/base-queue.d.ts +40 -0
- package/dist/storage/base-queue.d.ts.map +1 -0
- package/dist/storage/base-queue.js +89 -0
- package/dist/storage/base-queue.js.map +1 -0
- package/dist/storage/local-queue.d.ts +4 -24
- package/dist/storage/local-queue.d.ts.map +1 -1
- package/dist/storage/local-queue.js +5 -64
- package/dist/storage/local-queue.js.map +1 -1
- package/dist/storage/session-cursor-store.d.ts +14 -0
- package/dist/storage/session-cursor-store.d.ts.map +1 -0
- package/dist/storage/session-cursor-store.js +34 -0
- package/dist/storage/session-cursor-store.js.map +1 -0
- package/dist/storage/session-queue.d.ts +10 -0
- package/dist/storage/session-queue.d.ts.map +1 -0
- package/dist/storage/session-queue.js +11 -0
- package/dist/storage/session-queue.js.map +1 -0
- package/dist/utils/paths.d.ts +4 -0
- package/dist/utils/paths.d.ts.map +1 -1
- package/dist/utils/paths.js +5 -0
- package/dist/utils/paths.js.map +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import { Database } from "bun:sqlite";
|
|
2
|
+
/**
|
|
3
|
+
* Open an OpenCode SQLite database in read-only mode
|
|
4
|
+
* and return a queryMessages function for use with parseOpenCodeSqlite().
|
|
5
|
+
*
|
|
6
|
+
* Uses bun:sqlite for zero-dependency SQLite access.
|
|
7
|
+
* Returns null if the database cannot be opened.
|
|
8
|
+
*/
|
|
9
|
+
export function openMessageDb(dbPath) {
|
|
10
|
+
let db;
|
|
11
|
+
try {
|
|
12
|
+
db = new Database(dbPath, { readonly: true });
|
|
13
|
+
}
|
|
14
|
+
catch {
|
|
15
|
+
return null;
|
|
16
|
+
}
|
|
17
|
+
const stmt = db.query(`SELECT id, session_id, time_created, json_extract(data, '$.role') as role, data
|
|
18
|
+
FROM message
|
|
19
|
+
WHERE time_created >= ?
|
|
20
|
+
ORDER BY time_created ASC`);
|
|
21
|
+
return {
|
|
22
|
+
queryMessages: (lastTimeCreated) => stmt.all(lastTimeCreated),
|
|
23
|
+
close: () => db.close(),
|
|
24
|
+
};
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Open an OpenCode SQLite database in read-only mode
|
|
28
|
+
* and return session query functions for use with collectOpenCodeSqliteSessions().
|
|
29
|
+
*
|
|
30
|
+
* Returns null if the database cannot be opened.
|
|
31
|
+
*/
|
|
32
|
+
export function openSessionDb(dbPath) {
|
|
33
|
+
let db;
|
|
34
|
+
try {
|
|
35
|
+
db = new Database(dbPath, { readonly: true });
|
|
36
|
+
}
|
|
37
|
+
catch {
|
|
38
|
+
return null;
|
|
39
|
+
}
|
|
40
|
+
const sessionStmt = db.query(`SELECT id, project_id, title, time_created, time_updated
|
|
41
|
+
FROM session
|
|
42
|
+
WHERE time_updated >= ?
|
|
43
|
+
ORDER BY time_updated ASC`);
|
|
44
|
+
return {
|
|
45
|
+
querySessions: (lastTimeUpdated) => sessionStmt.all(lastTimeUpdated),
|
|
46
|
+
querySessionMessages: (sessionIds) => {
|
|
47
|
+
if (sessionIds.length === 0)
|
|
48
|
+
return [];
|
|
49
|
+
// SQLite has a 999 parameter limit. Batch session IDs into chunks
|
|
50
|
+
// of 500 to stay well under the limit.
|
|
51
|
+
const CHUNK_SIZE = 500;
|
|
52
|
+
const results = [];
|
|
53
|
+
for (let i = 0; i < sessionIds.length; i += CHUNK_SIZE) {
|
|
54
|
+
const chunk = sessionIds.slice(i, i + CHUNK_SIZE);
|
|
55
|
+
const placeholders = chunk.map(() => "?").join(",");
|
|
56
|
+
const stmt = db.query(`SELECT session_id, json_extract(data, '$.role') as role, time_created, data
|
|
57
|
+
FROM message
|
|
58
|
+
WHERE session_id IN (${placeholders})
|
|
59
|
+
ORDER BY time_created ASC`);
|
|
60
|
+
results.push(...stmt.all(...chunk));
|
|
61
|
+
}
|
|
62
|
+
// Re-sort across chunks to maintain global time_created order
|
|
63
|
+
if (sessionIds.length > CHUNK_SIZE) {
|
|
64
|
+
results.sort((a, b) => a.time_created - b.time_created);
|
|
65
|
+
}
|
|
66
|
+
return results;
|
|
67
|
+
},
|
|
68
|
+
close: () => db.close(),
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
//# sourceMappingURL=opencode-sqlite-db.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"opencode-sqlite-db.js","sourceRoot":"","sources":["../../src/parsers/opencode-sqlite-db.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAItC;;;;;;GAMG;AACH,MAAM,UAAU,aAAa,CAC3B,MAAc;IAEd,IAAI,EAAY,CAAC;IACjB,IAAI,CAAC;QACH,EAAE,GAAG,IAAI,QAAQ,CAAC,MAAM,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;IAChD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,IAAI,GAAG,EAAE,CAAC,KAAK,CACnB;;;+BAG2B,CAC5B,CAAC;IAEF,OAAO;QACL,aAAa,EAAE,CAAC,eAAuB,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,eAAe,CAAC;QACrE,KAAK,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,KAAK,EAAE;KACxB,CAAC;AACJ,CAAC;AAQD;;;;;GAKG;AACH,MAAM,UAAU,aAAa,CAC3B,MAAc;IAMd,IAAI,EAAY,CAAC;IACjB,IAAI,CAAC;QACH,EAAE,GAAG,IAAI,QAAQ,CAAC,MAAM,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;IAChD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,WAAW,GAAG,EAAE,CAAC,KAAK,CAC1B;;;+BAG2B,CAC5B,CAAC;IAEF,OAAO;QACL,aAAa,EAAE,CAAC,eAAuB,EAAE,EAAE,CACzC,WAAW,CAAC,GAAG,CAAC,eAAe,CAAC;QAElC,oBAAoB,EAAE,CAAC,UAAoB,EAAE,EAAE;YAC7C,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,EAAE,CAAC;YACvC,kEAAkE;YAClE,uCAAuC;YACvC,MAAM,UAAU,GAAG,GAAG,CAAC;YACvB,MAAM,OAAO,GAAwB,EAAE,CAAC;YACxC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,IAAI,UAAU,EAAE,CAAC;gBACvD,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,CAAC;gBAClD,MAAM,YAAY,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACpD,MAAM,IAAI,GAAG,EAAE,CAAC,KAAK,CACnB;;kCAEwB,YAAY;qCACT,CAC5B,CAAC;gBACF,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC;YACtC,CAAC;YACD,8DAA8D;YAC9D,IAAI,UAAU,CAAC,MAAM,GAAG,UAAU,EAAE,CAAC;gBACnC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC,YAAY,CAAC,CAAC;YAC1D,CAAC;YACD,OAAO,OAAO,CAAC;QACjB,CAAC;QAED,KAAK,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,KAAK,EAAE;KACxB,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* OpenCode SQLite session collector.
|
|
3
|
+
*
|
|
4
|
+
* Queries the `session` and `message` tables from opencode.db
|
|
5
|
+
* to produce SessionSnapshot records. Uses dependency injection
|
|
6
|
+
* (pre-fetched rows) for testability without bun:sqlite.
|
|
7
|
+
*/
|
|
8
|
+
import type { SessionSnapshot } from "@pew/core";
|
|
9
|
+
/** Row shape from the session table */
|
|
10
|
+
export interface SessionRow {
|
|
11
|
+
id: string;
|
|
12
|
+
project_id: string | null;
|
|
13
|
+
title: string | null;
|
|
14
|
+
time_created: number;
|
|
15
|
+
time_updated: number;
|
|
16
|
+
}
|
|
17
|
+
/** Row shape from message table for session collection (minimal) */
|
|
18
|
+
export interface SessionMessageRow {
|
|
19
|
+
session_id: string;
|
|
20
|
+
role: string;
|
|
21
|
+
time_created: number;
|
|
22
|
+
data: string;
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Collect session snapshots from pre-fetched SQLite rows.
|
|
26
|
+
*
|
|
27
|
+
* Groups messages by session, counts roles, tracks timestamps and model.
|
|
28
|
+
* Uses session table time_created/time_updated as fallback timestamps
|
|
29
|
+
* when message-level time is unavailable.
|
|
30
|
+
*/
|
|
31
|
+
export declare function collectOpenCodeSqliteSessions(sessions: SessionRow[], messages: SessionMessageRow[]): SessionSnapshot[];
|
|
32
|
+
//# sourceMappingURL=opencode-sqlite-session.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"opencode-sqlite-session.d.ts","sourceRoot":"","sources":["../../src/parsers/opencode-sqlite-session.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAU,MAAM,WAAW,CAAC;AAEzD,uCAAuC;AACvC,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,oEAAoE;AACpE,MAAM,WAAW,iBAAiB;IAChC,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;CACd;AAaD;;;;;;GAMG;AACH,wBAAgB,6BAA6B,CAC3C,QAAQ,EAAE,UAAU,EAAE,EACtB,QAAQ,EAAE,iBAAiB,EAAE,GAC5B,eAAe,EAAE,CAuGnB"}
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* OpenCode SQLite session collector.
|
|
3
|
+
*
|
|
4
|
+
* Queries the `session` and `message` tables from opencode.db
|
|
5
|
+
* to produce SessionSnapshot records. Uses dependency injection
|
|
6
|
+
* (pre-fetched rows) for testability without bun:sqlite.
|
|
7
|
+
*/
|
|
8
|
+
/**
|
|
9
|
+
* Coerce an epoch value to milliseconds.
|
|
10
|
+
* Values < 1e12 are treated as seconds and multiplied by 1000.
|
|
11
|
+
*/
|
|
12
|
+
function coerceEpochMs(v) {
|
|
13
|
+
const n = Number(v);
|
|
14
|
+
if (!Number.isFinite(n) || n <= 0)
|
|
15
|
+
return 0;
|
|
16
|
+
if (n < 1e12)
|
|
17
|
+
return Math.floor(n * 1000);
|
|
18
|
+
return Math.floor(n);
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Collect session snapshots from pre-fetched SQLite rows.
|
|
22
|
+
*
|
|
23
|
+
* Groups messages by session, counts roles, tracks timestamps and model.
|
|
24
|
+
* Uses session table time_created/time_updated as fallback timestamps
|
|
25
|
+
* when message-level time is unavailable.
|
|
26
|
+
*/
|
|
27
|
+
export function collectOpenCodeSqliteSessions(sessions, messages) {
|
|
28
|
+
if (sessions.length === 0)
|
|
29
|
+
return [];
|
|
30
|
+
// Group messages by session_id
|
|
31
|
+
const msgMap = new Map();
|
|
32
|
+
for (const msg of messages) {
|
|
33
|
+
const list = msgMap.get(msg.session_id);
|
|
34
|
+
if (list) {
|
|
35
|
+
list.push(msg);
|
|
36
|
+
}
|
|
37
|
+
else {
|
|
38
|
+
msgMap.set(msg.session_id, [msg]);
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
const snapshots = [];
|
|
42
|
+
for (const session of sessions) {
|
|
43
|
+
const sessionMessages = msgMap.get(session.id) ?? [];
|
|
44
|
+
let userMessages = 0;
|
|
45
|
+
let assistantMessages = 0;
|
|
46
|
+
const totalMessages = sessionMessages.length;
|
|
47
|
+
let minEpochMs = null;
|
|
48
|
+
let maxEpochMs = null;
|
|
49
|
+
let lastModel = null;
|
|
50
|
+
for (const msg of sessionMessages) {
|
|
51
|
+
// Count by role from the role column (faster than parsing JSON)
|
|
52
|
+
if (msg.role === "user") {
|
|
53
|
+
userMessages++;
|
|
54
|
+
}
|
|
55
|
+
else if (msg.role === "assistant") {
|
|
56
|
+
assistantMessages++;
|
|
57
|
+
}
|
|
58
|
+
// Parse data JSON for time and model extraction
|
|
59
|
+
let data = null;
|
|
60
|
+
try {
|
|
61
|
+
data = JSON.parse(msg.data);
|
|
62
|
+
}
|
|
63
|
+
catch {
|
|
64
|
+
// If data is corrupted, we still count the message but skip time/model
|
|
65
|
+
}
|
|
66
|
+
if (data) {
|
|
67
|
+
const time = data.time;
|
|
68
|
+
if (time) {
|
|
69
|
+
const completedMs = coerceEpochMs(time.completed);
|
|
70
|
+
const createdMs = coerceEpochMs(time.created);
|
|
71
|
+
const msgStart = createdMs || completedMs;
|
|
72
|
+
const msgEnd = completedMs || createdMs;
|
|
73
|
+
if (msgStart) {
|
|
74
|
+
if (!minEpochMs || msgStart < minEpochMs)
|
|
75
|
+
minEpochMs = msgStart;
|
|
76
|
+
}
|
|
77
|
+
if (msgEnd) {
|
|
78
|
+
if (!maxEpochMs || msgEnd > maxEpochMs)
|
|
79
|
+
maxEpochMs = msgEnd;
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
// Track model from assistant messages
|
|
83
|
+
const model = typeof data.modelID === "string"
|
|
84
|
+
? data.modelID.trim()
|
|
85
|
+
: typeof data.model === "string"
|
|
86
|
+
? data.model.trim()
|
|
87
|
+
: null;
|
|
88
|
+
if (model) {
|
|
89
|
+
lastModel = model;
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
// Fallback to session table timestamps if messages lack time
|
|
94
|
+
if (!minEpochMs) {
|
|
95
|
+
minEpochMs = coerceEpochMs(session.time_created) || null;
|
|
96
|
+
}
|
|
97
|
+
if (!maxEpochMs) {
|
|
98
|
+
maxEpochMs = coerceEpochMs(session.time_updated) || minEpochMs;
|
|
99
|
+
}
|
|
100
|
+
// Must have at least a start time
|
|
101
|
+
if (!minEpochMs)
|
|
102
|
+
continue;
|
|
103
|
+
const endMs = maxEpochMs ?? minEpochMs;
|
|
104
|
+
snapshots.push({
|
|
105
|
+
sessionKey: `opencode:${session.id}`,
|
|
106
|
+
source: "opencode",
|
|
107
|
+
kind: "human",
|
|
108
|
+
startedAt: new Date(minEpochMs).toISOString(),
|
|
109
|
+
lastMessageAt: new Date(endMs).toISOString(),
|
|
110
|
+
durationSeconds: Math.max(0, Math.floor((endMs - minEpochMs) / 1000)),
|
|
111
|
+
userMessages,
|
|
112
|
+
assistantMessages,
|
|
113
|
+
totalMessages,
|
|
114
|
+
projectRef: session.project_id ?? null,
|
|
115
|
+
model: lastModel,
|
|
116
|
+
snapshotAt: new Date().toISOString(),
|
|
117
|
+
});
|
|
118
|
+
}
|
|
119
|
+
return snapshots;
|
|
120
|
+
}
|
|
121
|
+
//# sourceMappingURL=opencode-sqlite-session.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"opencode-sqlite-session.js","sourceRoot":"","sources":["../../src/parsers/opencode-sqlite-session.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAqBH;;;GAGG;AACH,SAAS,aAAa,CAAC,CAAU;IAC/B,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;IACpB,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC;IAC5C,IAAI,CAAC,GAAG,IAAI;QAAE,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;IAC1C,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AACvB,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,6BAA6B,CAC3C,QAAsB,EACtB,QAA6B;IAE7B,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAErC,+BAA+B;IAC/B,MAAM,MAAM,GAAG,IAAI,GAAG,EAA+B,CAAC;IACtD,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QACxC,IAAI,IAAI,EAAE,CAAC;YACT,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACjB,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;QACpC,CAAC;IACH,CAAC;IAED,MAAM,SAAS,GAAsB,EAAE,CAAC;IAExC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,MAAM,eAAe,GAAG,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC;QAErD,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,IAAI,iBAAiB,GAAG,CAAC,CAAC;QAC1B,MAAM,aAAa,GAAG,eAAe,CAAC,MAAM,CAAC;QAE7C,IAAI,UAAU,GAAkB,IAAI,CAAC;QACrC,IAAI,UAAU,GAAkB,IAAI,CAAC;QACrC,IAAI,SAAS,GAAkB,IAAI,CAAC;QAEpC,KAAK,MAAM,GAAG,IAAI,eAAe,EAAE,CAAC;YAClC,gEAAgE;YAChE,IAAI,GAAG,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBACxB,YAAY,EAAE,CAAC;YACjB,CAAC;iBAAM,IAAI,GAAG,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;gBACpC,iBAAiB,EAAE,CAAC;YACtB,CAAC;YAED,gDAAgD;YAChD,IAAI,IAAI,GAAmC,IAAI,CAAC;YAChD,IAAI,CAAC;gBACH,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAC9B,CAAC;YAAC,MAAM,CAAC;gBACP,uEAAuE;YACzE,CAAC;YAED,IAAI,IAAI,EAAE,CAAC;gBACT,MAAM,IAAI,GAAG,IAAI,CAAC,IAA2C,CAAC;gBAC9D,IAAI,IAAI,EAAE,CAAC;oBACT,MAAM,WAAW,GAAG,aAAa,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;oBAClD,MAAM,SAAS,GAAG,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;oBAE9C,MAAM,QAAQ,GAAG,SAAS,IAAI,WAAW,CAAC;oBAC1C,MAAM,MAAM,GAAG,WAAW,IAAI,SAAS,CAAC;oBAExC,IAAI,QAAQ,EAAE,CAAC;wBACb,IAAI,CAAC,UAAU,IAAI,QAAQ,GAAG,UAAU;4BAAE,UAAU,GAAG,QAAQ,CAAC;oBAClE,CAAC;oBACD,IAAI,MAAM,EAAE,CAAC;wBACX,IAAI,CAAC,UAAU,IAAI,MAAM,GAAG,UAAU;4BAAE,UAAU,GAAG,MAAM,CAAC;oBAC9D,CAAC;gBACH,CAAC;gBAED,sCAAsC;gBACtC,MAAM,KAAK,GACT,OAAO,IAAI,CAAC,OAAO,KAAK,QAAQ;oBAC9B,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE;oBACrB,CAAC,CAAC,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ;wBAC9B,CAAC,CAAE,IAAI,CAAC,KAAgB,CAAC,IAAI,EAAE;wBAC/B,CAAC,CAAC,IAAI,CAAC;gBACb,IAAI,KAAK,EAAE,CAAC;oBACV,SAAS,GAAG,KAAK,CAAC;gBACpB,CAAC;YACH,CAAC;QACH,CAAC;QAED,6DAA6D;QAC7D,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,UAAU,GAAG,aAAa,CAAC,OAAO,CAAC,YAAY,CAAC,IAAI,IAAI,CAAC;QAC3D,CAAC;QACD,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,UAAU,GAAG,aAAa,CAAC,OAAO,CAAC,YAAY,CAAC,IAAI,UAAU,CAAC;QACjE,CAAC;QAED,kCAAkC;QAClC,IAAI,CAAC,UAAU;YAAE,SAAS;QAE1B,MAAM,KAAK,GAAG,UAAU,IAAI,UAAU,CAAC;QAEvC,SAAS,CAAC,IAAI,CAAC;YACb,UAAU,EAAE,YAAY,OAAO,CAAC,EAAE,EAAE;YACpC,MAAM,EAAE,UAAoB;YAC5B,IAAI,EAAE,OAAO;YACb,SAAS,EAAE,IAAI,IAAI,CAAC,UAAU,CAAC,CAAC,WAAW,EAAE;YAC7C,aAAa,EAAE,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE;YAC5C,eAAe,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,GAAG,UAAU,CAAC,GAAG,IAAI,CAAC,CAAC;YACrE,YAAY;YACZ,iBAAiB;YACjB,aAAa;YACb,UAAU,EAAE,OAAO,CAAC,UAAU,IAAI,IAAI;YACtC,KAAK,EAAE,SAAS;YAChB,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACrC,CAAC,CAAC;IACL,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC"}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import type { ParsedDelta } from "./claude.js";
|
|
2
|
+
/** Result of parsing OpenCode SQLite database */
|
|
3
|
+
export interface OpenCodeSqliteResult {
|
|
4
|
+
/** Parsed token deltas (one per assistant message) */
|
|
5
|
+
deltas: ParsedDelta[];
|
|
6
|
+
/** Message keys for dedup: "sessionId|messageId" */
|
|
7
|
+
messageKeys: Set<string>;
|
|
8
|
+
/** Highest time_created seen (for cursor advancement) */
|
|
9
|
+
maxTimeCreated: number;
|
|
10
|
+
/** DB file inode (for detecting replacement/recreation) */
|
|
11
|
+
inode: number;
|
|
12
|
+
}
|
|
13
|
+
/** Row shape from the message table */
|
|
14
|
+
export interface MessageRow {
|
|
15
|
+
id: string;
|
|
16
|
+
session_id: string;
|
|
17
|
+
time_created: number;
|
|
18
|
+
/** Extracted via json_extract(data, '$.role') at the SQL level */
|
|
19
|
+
role: string | null;
|
|
20
|
+
data: string;
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Function that queries the message table.
|
|
24
|
+
* Accepts lastTimeCreated and returns rows where time_created >= lastTimeCreated.
|
|
25
|
+
* Callers must filter out previously-processed IDs from the prior batch
|
|
26
|
+
* to handle same-millisecond boundary dedup.
|
|
27
|
+
*/
|
|
28
|
+
export type QueryMessagesFn = (lastTimeCreated: number) => MessageRow[];
|
|
29
|
+
/**
|
|
30
|
+
* Parse message rows from OpenCode's SQLite database for token usage records.
|
|
31
|
+
*
|
|
32
|
+
* Processes rows from the `message` table where `time_created > lastTimeCreated`.
|
|
33
|
+
*
|
|
34
|
+
* Unlike the JSON file parser, no diffTotals is needed — each SQLite row
|
|
35
|
+
* is an independent message with absolute token values.
|
|
36
|
+
*
|
|
37
|
+
* The `queryMessages` function is injected to decouple from `bun:sqlite`
|
|
38
|
+
* for testability. Use `openMessageDb()` from `opencode-sqlite-db.ts` to
|
|
39
|
+
* create the real adapter at runtime.
|
|
40
|
+
*/
|
|
41
|
+
export declare function processOpenCodeMessages(rows: MessageRow[]): Omit<OpenCodeSqliteResult, "inode">;
|
|
42
|
+
/**
|
|
43
|
+
* High-level entry: parse OpenCode SQLite database for token usage.
|
|
44
|
+
*
|
|
45
|
+
* Opens the database, queries new messages, and returns parsed deltas.
|
|
46
|
+
* The `queryMessages` function provides the database access layer.
|
|
47
|
+
*/
|
|
48
|
+
export declare function parseOpenCodeSqlite(opts: {
|
|
49
|
+
dbPath: string;
|
|
50
|
+
lastTimeCreated: number;
|
|
51
|
+
queryMessages?: QueryMessagesFn;
|
|
52
|
+
}): Promise<OpenCodeSqliteResult>;
|
|
53
|
+
//# sourceMappingURL=opencode-sqlite.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"opencode-sqlite.d.ts","sourceRoot":"","sources":["../../src/parsers/opencode-sqlite.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAG/C,iDAAiD;AACjD,MAAM,WAAW,oBAAoB;IACnC,sDAAsD;IACtD,MAAM,EAAE,WAAW,EAAE,CAAC;IACtB,oDAAoD;IACpD,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IACzB,yDAAyD;IACzD,cAAc,EAAE,MAAM,CAAC;IACvB,2DAA2D;IAC3D,KAAK,EAAE,MAAM,CAAC;CACf;AAED,uCAAuC;AACvC,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,kEAAkE;IAClE,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;CACd;AAED;;;;;GAKG;AACH,MAAM,MAAM,eAAe,GAAG,CAAC,eAAe,EAAE,MAAM,KAAK,UAAU,EAAE,CAAC;AAiBxE;;;;;;;;;;;GAWG;AACH,wBAAgB,uBAAuB,CACrC,IAAI,EAAE,UAAU,EAAE,GACjB,IAAI,CAAC,oBAAoB,EAAE,OAAO,CAAC,CAsDrC;AAED;;;;;GAKG;AACH,wBAAsB,mBAAmB,CAAC,IAAI,EAAE;IAC9C,MAAM,EAAE,MAAM,CAAC;IACf,eAAe,EAAE,MAAM,CAAC;IACxB,aAAa,CAAC,EAAE,eAAe,CAAC;CACjC,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAgChC"}
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
import { stat } from "node:fs/promises";
|
|
2
|
+
import { normalizeOpenCodeTokens, coerceEpochMs } from "./opencode.js";
|
|
3
|
+
/** Check if a TokenDelta is all zeros */
|
|
4
|
+
function isAllZero(d) {
|
|
5
|
+
return (d.inputTokens === 0 &&
|
|
6
|
+
d.cachedInputTokens === 0 &&
|
|
7
|
+
d.outputTokens === 0 &&
|
|
8
|
+
d.reasoningOutputTokens === 0);
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* Parse message rows from OpenCode's SQLite database for token usage records.
|
|
12
|
+
*
|
|
13
|
+
* Processes rows from the `message` table where `time_created > lastTimeCreated`.
|
|
14
|
+
*
|
|
15
|
+
* Unlike the JSON file parser, no diffTotals is needed — each SQLite row
|
|
16
|
+
* is an independent message with absolute token values.
|
|
17
|
+
*
|
|
18
|
+
* The `queryMessages` function is injected to decouple from `bun:sqlite`
|
|
19
|
+
* for testability. Use `openMessageDb()` from `opencode-sqlite-db.ts` to
|
|
20
|
+
* create the real adapter at runtime.
|
|
21
|
+
*/
|
|
22
|
+
export function processOpenCodeMessages(rows) {
|
|
23
|
+
const deltas = [];
|
|
24
|
+
const messageKeys = new Set();
|
|
25
|
+
let maxTimeCreated = 0;
|
|
26
|
+
for (const row of rows) {
|
|
27
|
+
// Track max time_created for cursor
|
|
28
|
+
if (row.time_created > maxTimeCreated) {
|
|
29
|
+
maxTimeCreated = row.time_created;
|
|
30
|
+
}
|
|
31
|
+
// Only process assistant messages (role is pre-extracted at SQL level)
|
|
32
|
+
if (row.role !== "assistant")
|
|
33
|
+
continue;
|
|
34
|
+
// Parse data JSON
|
|
35
|
+
let msg;
|
|
36
|
+
try {
|
|
37
|
+
msg = JSON.parse(row.data);
|
|
38
|
+
}
|
|
39
|
+
catch {
|
|
40
|
+
continue;
|
|
41
|
+
}
|
|
42
|
+
// Build message key for dedup
|
|
43
|
+
messageKeys.add(`${row.session_id}|${row.id}`);
|
|
44
|
+
// Normalize tokens
|
|
45
|
+
const tokens = normalizeOpenCodeTokens(msg.tokens);
|
|
46
|
+
if (!tokens || isAllZero(tokens))
|
|
47
|
+
continue;
|
|
48
|
+
// Extract timestamp from time.completed or time.created
|
|
49
|
+
const time = msg.time;
|
|
50
|
+
const timestampMs = coerceEpochMs(time?.completed) || coerceEpochMs(time?.created);
|
|
51
|
+
if (!timestampMs)
|
|
52
|
+
continue;
|
|
53
|
+
// Extract model
|
|
54
|
+
const model = typeof msg.modelID === "string"
|
|
55
|
+
? msg.modelID.trim()
|
|
56
|
+
: typeof msg.model === "string"
|
|
57
|
+
? msg.model.trim()
|
|
58
|
+
: "unknown";
|
|
59
|
+
deltas.push({
|
|
60
|
+
source: "opencode",
|
|
61
|
+
model,
|
|
62
|
+
timestamp: new Date(timestampMs).toISOString(),
|
|
63
|
+
tokens,
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
return { deltas, messageKeys, maxTimeCreated };
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* High-level entry: parse OpenCode SQLite database for token usage.
|
|
70
|
+
*
|
|
71
|
+
* Opens the database, queries new messages, and returns parsed deltas.
|
|
72
|
+
* The `queryMessages` function provides the database access layer.
|
|
73
|
+
*/
|
|
74
|
+
export async function parseOpenCodeSqlite(opts) {
|
|
75
|
+
const { dbPath, lastTimeCreated, queryMessages } = opts;
|
|
76
|
+
const empty = {
|
|
77
|
+
deltas: [],
|
|
78
|
+
messageKeys: new Set(),
|
|
79
|
+
maxTimeCreated: 0,
|
|
80
|
+
inode: 0,
|
|
81
|
+
};
|
|
82
|
+
// Check if database file exists and get inode
|
|
83
|
+
let fileInode;
|
|
84
|
+
try {
|
|
85
|
+
const st = await stat(dbPath);
|
|
86
|
+
fileInode = st.ino;
|
|
87
|
+
}
|
|
88
|
+
catch {
|
|
89
|
+
return empty;
|
|
90
|
+
}
|
|
91
|
+
if (!queryMessages) {
|
|
92
|
+
return empty;
|
|
93
|
+
}
|
|
94
|
+
let rows;
|
|
95
|
+
try {
|
|
96
|
+
rows = queryMessages(lastTimeCreated);
|
|
97
|
+
}
|
|
98
|
+
catch {
|
|
99
|
+
return empty;
|
|
100
|
+
}
|
|
101
|
+
const result = processOpenCodeMessages(rows);
|
|
102
|
+
return { ...result, inode: fileInode };
|
|
103
|
+
}
|
|
104
|
+
//# sourceMappingURL=opencode-sqlite.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"opencode-sqlite.js","sourceRoot":"","sources":["../../src/parsers/opencode-sqlite.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAGxC,OAAO,EAAE,uBAAuB,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAgCvE,yCAAyC;AACzC,SAAS,SAAS,CAAC,CAKlB;IACC,OAAO,CACL,CAAC,CAAC,WAAW,KAAK,CAAC;QACnB,CAAC,CAAC,iBAAiB,KAAK,CAAC;QACzB,CAAC,CAAC,YAAY,KAAK,CAAC;QACpB,CAAC,CAAC,qBAAqB,KAAK,CAAC,CAC9B,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,uBAAuB,CACrC,IAAkB;IAElB,MAAM,MAAM,GAAkB,EAAE,CAAC;IACjC,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;IACtC,IAAI,cAAc,GAAG,CAAC,CAAC;IAEvB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,oCAAoC;QACpC,IAAI,GAAG,CAAC,YAAY,GAAG,cAAc,EAAE,CAAC;YACtC,cAAc,GAAG,GAAG,CAAC,YAAY,CAAC;QACpC,CAAC;QAED,uEAAuE;QACvE,IAAI,GAAG,CAAC,IAAI,KAAK,WAAW;YAAE,SAAS;QAEvC,kBAAkB;QAClB,IAAI,GAA4B,CAAC;QACjC,IAAI,CAAC;YACH,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC7B,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;QAED,8BAA8B;QAC9B,WAAW,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,UAAU,IAAI,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC;QAE/C,mBAAmB;QACnB,MAAM,MAAM,GAAG,uBAAuB,CACpC,GAAG,CAAC,MAAwC,CAC7C,CAAC;QACF,IAAI,CAAC,MAAM,IAAI,SAAS,CAAC,MAAM,CAAC;YAAE,SAAS;QAE3C,wDAAwD;QACxD,MAAM,IAAI,GAAG,GAAG,CAAC,IAA2C,CAAC;QAC7D,MAAM,WAAW,GACf,aAAa,CAAC,IAAI,EAAE,SAAS,CAAC,IAAI,aAAa,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACjE,IAAI,CAAC,WAAW;YAAE,SAAS;QAE3B,gBAAgB;QAChB,MAAM,KAAK,GACT,OAAO,GAAG,CAAC,OAAO,KAAK,QAAQ;YAC7B,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE;YACpB,CAAC,CAAC,OAAO,GAAG,CAAC,KAAK,KAAK,QAAQ;gBAC7B,CAAC,CAAE,GAAG,CAAC,KAAgB,CAAC,IAAI,EAAE;gBAC9B,CAAC,CAAC,SAAS,CAAC;QAElB,MAAM,CAAC,IAAI,CAAC;YACV,MAAM,EAAE,UAAoB;YAC5B,KAAK;YACL,SAAS,EAAE,IAAI,IAAI,CAAC,WAAW,CAAC,CAAC,WAAW,EAAE;YAC9C,MAAM;SACP,CAAC,CAAC;IACL,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,cAAc,EAAE,CAAC;AACjD,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,IAIzC;IACC,MAAM,EAAE,MAAM,EAAE,eAAe,EAAE,aAAa,EAAE,GAAG,IAAI,CAAC;IAExD,MAAM,KAAK,GAAyB;QAClC,MAAM,EAAE,EAAE;QACV,WAAW,EAAE,IAAI,GAAG,EAAE;QACtB,cAAc,EAAE,CAAC;QACjB,KAAK,EAAE,CAAC;KACT,CAAC;IAEF,8CAA8C;IAC9C,IAAI,SAAiB,CAAC;IACtB,IAAI,CAAC;QACH,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC;QAC9B,SAAS,GAAG,EAAE,CAAC,GAAG,CAAC;IACrB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,CAAC,aAAa,EAAE,CAAC;QACnB,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,IAAkB,CAAC;IACvB,IAAI,CAAC;QACH,IAAI,GAAG,aAAa,CAAC,eAAe,CAAC,CAAC;IACxC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;IAED,MAAM,MAAM,GAAG,uBAAuB,CAAC,IAAI,CAAC,CAAC;IAC7C,OAAO,EAAE,GAAG,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;AACzC,CAAC"}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Generic append-only JSONL queue with byte-offset tracking.
|
|
3
|
+
*
|
|
4
|
+
* Records are stored as newline-delimited JSON. A separate state file
|
|
5
|
+
* tracks how far the upload cursor has progressed (in bytes).
|
|
6
|
+
*
|
|
7
|
+
* Bug fixes over the original LocalQueue/SessionQueue:
|
|
8
|
+
* - readFromOffset uses Buffer slicing (byte offset) instead of String.slice
|
|
9
|
+
* (character offset), which broke on non-ASCII content.
|
|
10
|
+
* - Per-line JSON.parse error handling: corrupted lines are skipped instead
|
|
11
|
+
* of blocking all subsequent records.
|
|
12
|
+
*/
|
|
13
|
+
export declare class BaseQueue<T> {
|
|
14
|
+
readonly queuePath: string;
|
|
15
|
+
private readonly statePath;
|
|
16
|
+
private readonly dir;
|
|
17
|
+
constructor(storeDir: string, queueFile: string, stateFile: string);
|
|
18
|
+
/** Ensure the directory exists */
|
|
19
|
+
private ensureDir;
|
|
20
|
+
/** Append a single record to the queue */
|
|
21
|
+
append(record: T): Promise<void>;
|
|
22
|
+
/** Append multiple records to the queue in a single write */
|
|
23
|
+
appendBatch(records: T[]): Promise<void>;
|
|
24
|
+
/**
|
|
25
|
+
* Read records from the queue starting at a byte offset.
|
|
26
|
+
* Returns parsed records and the new offset (end of file in bytes).
|
|
27
|
+
*
|
|
28
|
+
* - Uses Buffer for slicing to correctly handle multi-byte UTF-8 characters.
|
|
29
|
+
* - Skips lines that fail JSON.parse instead of throwing.
|
|
30
|
+
*/
|
|
31
|
+
readFromOffset(offset: number): Promise<{
|
|
32
|
+
records: T[];
|
|
33
|
+
newOffset: number;
|
|
34
|
+
}>;
|
|
35
|
+
/** Save the upload byte offset to the state file */
|
|
36
|
+
saveOffset(offset: number): Promise<void>;
|
|
37
|
+
/** Load the upload byte offset. Returns 0 if not found or corrupted. */
|
|
38
|
+
loadOffset(): Promise<number>;
|
|
39
|
+
}
|
|
40
|
+
//# sourceMappingURL=base-queue.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"base-queue.d.ts","sourceRoot":"","sources":["../../src/storage/base-queue.ts"],"names":[],"mappings":"AAGA;;;;;;;;;;;GAWG;AACH,qBAAa,SAAS,CAAC,CAAC;IACtB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAS;gBAEjB,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM;IAMlE,kCAAkC;YACpB,SAAS;IAIvB,0CAA0C;IACpC,MAAM,CAAC,MAAM,EAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAKtC,6DAA6D;IACvD,WAAW,CAAC,OAAO,EAAE,CAAC,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAO9C;;;;;;OAMG;IACG,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC;QAC5C,OAAO,EAAE,CAAC,EAAE,CAAC;QACb,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IA0BF,oDAAoD;IAC9C,UAAU,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAK/C,wEAAwE;IAClE,UAAU,IAAI,OAAO,CAAC,MAAM,CAAC;CASpC"}
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
import { readFile, writeFile, appendFile, mkdir } from "node:fs/promises";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
/**
|
|
4
|
+
* Generic append-only JSONL queue with byte-offset tracking.
|
|
5
|
+
*
|
|
6
|
+
* Records are stored as newline-delimited JSON. A separate state file
|
|
7
|
+
* tracks how far the upload cursor has progressed (in bytes).
|
|
8
|
+
*
|
|
9
|
+
* Bug fixes over the original LocalQueue/SessionQueue:
|
|
10
|
+
* - readFromOffset uses Buffer slicing (byte offset) instead of String.slice
|
|
11
|
+
* (character offset), which broke on non-ASCII content.
|
|
12
|
+
* - Per-line JSON.parse error handling: corrupted lines are skipped instead
|
|
13
|
+
* of blocking all subsequent records.
|
|
14
|
+
*/
|
|
15
|
+
export class BaseQueue {
|
|
16
|
+
queuePath;
|
|
17
|
+
statePath;
|
|
18
|
+
dir;
|
|
19
|
+
constructor(storeDir, queueFile, stateFile) {
|
|
20
|
+
this.dir = storeDir;
|
|
21
|
+
this.queuePath = join(storeDir, queueFile);
|
|
22
|
+
this.statePath = join(storeDir, stateFile);
|
|
23
|
+
}
|
|
24
|
+
/** Ensure the directory exists */
|
|
25
|
+
async ensureDir() {
|
|
26
|
+
await mkdir(this.dir, { recursive: true });
|
|
27
|
+
}
|
|
28
|
+
/** Append a single record to the queue */
|
|
29
|
+
async append(record) {
|
|
30
|
+
await this.ensureDir();
|
|
31
|
+
await appendFile(this.queuePath, JSON.stringify(record) + "\n");
|
|
32
|
+
}
|
|
33
|
+
/** Append multiple records to the queue in a single write */
|
|
34
|
+
async appendBatch(records) {
|
|
35
|
+
if (records.length === 0)
|
|
36
|
+
return;
|
|
37
|
+
await this.ensureDir();
|
|
38
|
+
const data = records.map((r) => JSON.stringify(r)).join("\n") + "\n";
|
|
39
|
+
await appendFile(this.queuePath, data);
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Read records from the queue starting at a byte offset.
|
|
43
|
+
* Returns parsed records and the new offset (end of file in bytes).
|
|
44
|
+
*
|
|
45
|
+
* - Uses Buffer for slicing to correctly handle multi-byte UTF-8 characters.
|
|
46
|
+
* - Skips lines that fail JSON.parse instead of throwing.
|
|
47
|
+
*/
|
|
48
|
+
async readFromOffset(offset) {
|
|
49
|
+
let buf;
|
|
50
|
+
try {
|
|
51
|
+
buf = await readFile(this.queuePath);
|
|
52
|
+
}
|
|
53
|
+
catch {
|
|
54
|
+
return { records: [], newOffset: 0 };
|
|
55
|
+
}
|
|
56
|
+
const slice = buf.subarray(offset);
|
|
57
|
+
const text = slice.toString("utf-8");
|
|
58
|
+
const lines = text.split("\n").filter((line) => line.trim().length > 0);
|
|
59
|
+
const records = [];
|
|
60
|
+
for (const line of lines) {
|
|
61
|
+
try {
|
|
62
|
+
records.push(JSON.parse(line));
|
|
63
|
+
}
|
|
64
|
+
catch {
|
|
65
|
+
// Skip corrupted lines — log would be nice but we keep it silent
|
|
66
|
+
// to avoid coupling to a logger. The line is simply lost.
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
const newOffset = buf.byteLength;
|
|
70
|
+
return { records, newOffset };
|
|
71
|
+
}
|
|
72
|
+
/** Save the upload byte offset to the state file */
|
|
73
|
+
async saveOffset(offset) {
|
|
74
|
+
await this.ensureDir();
|
|
75
|
+
await writeFile(this.statePath, JSON.stringify({ offset }) + "\n");
|
|
76
|
+
}
|
|
77
|
+
/** Load the upload byte offset. Returns 0 if not found or corrupted. */
|
|
78
|
+
async loadOffset() {
|
|
79
|
+
try {
|
|
80
|
+
const raw = await readFile(this.statePath, "utf-8");
|
|
81
|
+
const state = JSON.parse(raw);
|
|
82
|
+
return state.offset ?? 0;
|
|
83
|
+
}
|
|
84
|
+
catch {
|
|
85
|
+
return 0;
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
//# sourceMappingURL=base-queue.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"base-queue.js","sourceRoot":"","sources":["../../src/storage/base-queue.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC1E,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC;;;;;;;;;;;GAWG;AACH,MAAM,OAAO,SAAS;IACX,SAAS,CAAS;IACV,SAAS,CAAS;IAClB,GAAG,CAAS;IAE7B,YAAY,QAAgB,EAAE,SAAiB,EAAE,SAAiB;QAChE,IAAI,CAAC,GAAG,GAAG,QAAQ,CAAC;QACpB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;QAC3C,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;IAC7C,CAAC;IAED,kCAAkC;IAC1B,KAAK,CAAC,SAAS;QACrB,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC7C,CAAC;IAED,0CAA0C;IAC1C,KAAK,CAAC,MAAM,CAAC,MAAS;QACpB,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;QACvB,MAAM,UAAU,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC;IAClE,CAAC;IAED,6DAA6D;IAC7D,KAAK,CAAC,WAAW,CAAC,OAAY;QAC5B,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QACjC,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;QACvB,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;QACrE,MAAM,UAAU,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IACzC,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,cAAc,CAAC,MAAc;QAIjC,IAAI,GAAW,CAAC;QAChB,IAAI,CAAC;YACH,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACvC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;QACvC,CAAC;QAED,MAAM,KAAK,GAAG,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACnC,MAAM,IAAI,GAAG,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QACrC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAExE,MAAM,OAAO,GAAQ,EAAE,CAAC;QACxB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,CAAC;gBACH,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAM,CAAC,CAAC;YACtC,CAAC;YAAC,MAAM,CAAC;gBACP,iEAAiE;gBACjE,0DAA0D;YAC5D,CAAC;QACH,CAAC;QAED,MAAM,SAAS,GAAG,GAAG,CAAC,UAAU,CAAC;QACjC,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC;IAChC,CAAC;IAED,oDAAoD;IACpD,KAAK,CAAC,UAAU,CAAC,MAAc;QAC7B,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;QACvB,MAAM,SAAS,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;IACrE,CAAC;IAED,wEAAwE;IACxE,KAAK,CAAC,UAAU;QACd,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;YACpD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAuB,CAAC;YACpD,OAAO,KAAK,CAAC,MAAM,IAAI,CAAC,CAAC;QAC3B,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,CAAC,CAAC;QACX,CAAC;IACH,CAAC;CACF"}
|
|
@@ -1,30 +1,10 @@
|
|
|
1
|
+
import { BaseQueue } from "./base-queue.js";
|
|
1
2
|
import type { QueueRecord } from "@pew/core";
|
|
2
3
|
/**
|
|
3
|
-
* Append-only local queue for usage records.
|
|
4
|
-
*
|
|
4
|
+
* Append-only local queue for token usage records.
|
|
5
|
+
* Thin wrapper around BaseQueue with token-specific file names.
|
|
5
6
|
*/
|
|
6
|
-
export declare class LocalQueue {
|
|
7
|
-
readonly queuePath: string;
|
|
8
|
-
private readonly statePath;
|
|
9
|
-
private readonly dir;
|
|
7
|
+
export declare class LocalQueue extends BaseQueue<QueueRecord> {
|
|
10
8
|
constructor(storeDir: string);
|
|
11
|
-
/** Ensure the directory exists */
|
|
12
|
-
private ensureDir;
|
|
13
|
-
/** Append a single record to the queue */
|
|
14
|
-
append(record: QueueRecord): Promise<void>;
|
|
15
|
-
/** Append multiple records to the queue in a single write */
|
|
16
|
-
appendBatch(records: QueueRecord[]): Promise<void>;
|
|
17
|
-
/**
|
|
18
|
-
* Read records from the queue starting at a byte offset.
|
|
19
|
-
* Returns parsed records and the new offset (end of file).
|
|
20
|
-
*/
|
|
21
|
-
readFromOffset(offset: number): Promise<{
|
|
22
|
-
records: QueueRecord[];
|
|
23
|
-
newOffset: number;
|
|
24
|
-
}>;
|
|
25
|
-
/** Save the upload byte offset to the state file */
|
|
26
|
-
saveOffset(offset: number): Promise<void>;
|
|
27
|
-
/** Load the upload byte offset. Returns 0 if not found or corrupted. */
|
|
28
|
-
loadOffset(): Promise<number>;
|
|
29
9
|
}
|
|
30
10
|
//# sourceMappingURL=local-queue.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"local-queue.d.ts","sourceRoot":"","sources":["../../src/storage/local-queue.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"local-queue.d.ts","sourceRoot":"","sources":["../../src/storage/local-queue.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAE7C;;;GAGG;AACH,qBAAa,UAAW,SAAQ,SAAS,CAAC,WAAW,CAAC;gBACxC,QAAQ,EAAE,MAAM;CAG7B"}
|