@hasna/economy 0.2.26 → 0.2.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/index.js +440 -81
- package/dist/db/database.d.ts +1 -1
- package/dist/db/database.d.ts.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +444 -86
- package/dist/lib/peer-sync.d.ts +21 -0
- package/dist/lib/peer-sync.d.ts.map +1 -0
- package/dist/mcp/index.js +132 -68
- package/dist/otel/index.js +35 -43
- package/dist/server/index.js +133 -69
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -565,6 +565,26 @@ function openDatabase(dbPath, skipSeed = false) {
|
|
|
565
565
|
}
|
|
566
566
|
return db;
|
|
567
567
|
}
|
|
568
|
+
function quoteSqlIdent(identifier) {
|
|
569
|
+
return `"${identifier.replace(/"/g, '""')}"`;
|
|
570
|
+
}
|
|
571
|
+
function hasColumn(db, table, column) {
|
|
572
|
+
const columns = db.prepare(`PRAGMA table_info(${quoteSqlIdent(table)})`).all();
|
|
573
|
+
return columns.some((c) => c.name === column);
|
|
574
|
+
}
|
|
575
|
+
function addColumnIfMissing(db, table, column, definition) {
|
|
576
|
+
if (hasColumn(db, table, column))
|
|
577
|
+
return false;
|
|
578
|
+
try {
|
|
579
|
+
db.exec(`ALTER TABLE ${quoteSqlIdent(table)} ADD COLUMN ${quoteSqlIdent(column)} ${definition}`);
|
|
580
|
+
return true;
|
|
581
|
+
} catch (error) {
|
|
582
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
583
|
+
if (/duplicate column name/i.test(message))
|
|
584
|
+
return true;
|
|
585
|
+
throw error;
|
|
586
|
+
}
|
|
587
|
+
}
|
|
568
588
|
function initSchema(db) {
|
|
569
589
|
db.exec(`
|
|
570
590
|
CREATE TABLE IF NOT EXISTS requests (
|
|
@@ -735,59 +755,31 @@ function initSchema(db) {
|
|
|
735
755
|
CREATE INDEX IF NOT EXISTS idx_usage_agent_date ON usage_snapshots(agent, date);
|
|
736
756
|
CREATE INDEX IF NOT EXISTS idx_savings_date ON savings_daily(date);
|
|
737
757
|
`);
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
db.exec(`ALTER TABLE sessions ADD COLUMN machine_id TEXT DEFAULT ''`);
|
|
742
|
-
}
|
|
743
|
-
if (!cols.some((c) => c.name === "cache_create_5m_tokens")) {
|
|
744
|
-
db.exec(`ALTER TABLE requests ADD COLUMN cache_create_5m_tokens INTEGER DEFAULT 0`);
|
|
758
|
+
addColumnIfMissing(db, "requests", "machine_id", `TEXT DEFAULT ''`);
|
|
759
|
+
addColumnIfMissing(db, "sessions", "machine_id", `TEXT DEFAULT ''`);
|
|
760
|
+
if (addColumnIfMissing(db, "requests", "cache_create_5m_tokens", "INTEGER DEFAULT 0")) {
|
|
745
761
|
db.exec(`UPDATE requests SET cache_create_5m_tokens = cache_create_tokens WHERE cache_create_5m_tokens = 0`);
|
|
746
762
|
}
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
if (
|
|
751
|
-
db.exec(`ALTER TABLE requests ADD COLUMN cost_basis TEXT DEFAULT 'estimated'`);
|
|
752
|
-
}
|
|
753
|
-
if (!cols.some((c) => c.name === "attribution_tag")) {
|
|
754
|
-
db.exec(`ALTER TABLE requests ADD COLUMN attribution_tag TEXT DEFAULT ''`);
|
|
755
|
-
}
|
|
756
|
-
if (!cols.some((c) => c.name === "updated_at")) {
|
|
757
|
-
db.exec(`ALTER TABLE requests ADD COLUMN updated_at TEXT DEFAULT ''`);
|
|
763
|
+
addColumnIfMissing(db, "requests", "cache_create_1h_tokens", "INTEGER DEFAULT 0");
|
|
764
|
+
addColumnIfMissing(db, "requests", "cost_basis", `TEXT DEFAULT 'estimated'`);
|
|
765
|
+
addColumnIfMissing(db, "requests", "attribution_tag", `TEXT DEFAULT ''`);
|
|
766
|
+
if (addColumnIfMissing(db, "requests", "updated_at", `TEXT DEFAULT ''`)) {
|
|
758
767
|
db.exec(`UPDATE requests SET updated_at = timestamp WHERE updated_at = '' OR updated_at IS NULL`);
|
|
759
768
|
}
|
|
760
|
-
|
|
761
|
-
db.exec(`ALTER TABLE requests ADD COLUMN synced_at TEXT DEFAULT ''`);
|
|
762
|
-
}
|
|
769
|
+
addColumnIfMissing(db, "requests", "synced_at", `TEXT DEFAULT ''`);
|
|
763
770
|
for (const column of ["account_key", "account_tool", "account_name", "account_email", "account_source"]) {
|
|
764
|
-
|
|
765
|
-
db.exec(`ALTER TABLE requests ADD COLUMN ${column} TEXT DEFAULT ''`);
|
|
766
|
-
}
|
|
767
|
-
}
|
|
768
|
-
const sessionCols = db.prepare(`PRAGMA table_info(sessions)`).all();
|
|
769
|
-
if (!sessionCols.some((c) => c.name === "attribution_tag")) {
|
|
770
|
-
db.exec(`ALTER TABLE sessions ADD COLUMN attribution_tag TEXT DEFAULT ''`);
|
|
771
|
+
addColumnIfMissing(db, "requests", column, `TEXT DEFAULT ''`);
|
|
771
772
|
}
|
|
772
|
-
|
|
773
|
-
|
|
773
|
+
addColumnIfMissing(db, "sessions", "attribution_tag", `TEXT DEFAULT ''`);
|
|
774
|
+
if (addColumnIfMissing(db, "sessions", "updated_at", `TEXT DEFAULT ''`)) {
|
|
774
775
|
db.exec(`UPDATE sessions SET updated_at = started_at WHERE updated_at = '' OR updated_at IS NULL`);
|
|
775
776
|
}
|
|
776
|
-
|
|
777
|
-
db.exec(`ALTER TABLE sessions ADD COLUMN synced_at TEXT DEFAULT ''`);
|
|
778
|
-
}
|
|
777
|
+
addColumnIfMissing(db, "sessions", "synced_at", `TEXT DEFAULT ''`);
|
|
779
778
|
for (const column of ["account_key", "account_tool", "account_name", "account_email", "account_source"]) {
|
|
780
|
-
|
|
781
|
-
db.exec(`ALTER TABLE sessions ADD COLUMN ${column} TEXT DEFAULT ''`);
|
|
782
|
-
}
|
|
783
|
-
}
|
|
784
|
-
const pricingCols = db.prepare(`PRAGMA table_info(model_pricing)`).all();
|
|
785
|
-
if (!pricingCols.some((c) => c.name === "cache_write_1h_per_1m")) {
|
|
786
|
-
db.exec(`ALTER TABLE model_pricing ADD COLUMN cache_write_1h_per_1m REAL NOT NULL DEFAULT 0`);
|
|
787
|
-
}
|
|
788
|
-
if (!pricingCols.some((c) => c.name === "cache_storage_per_1m_hour")) {
|
|
789
|
-
db.exec(`ALTER TABLE model_pricing ADD COLUMN cache_storage_per_1m_hour REAL NOT NULL DEFAULT 0`);
|
|
779
|
+
addColumnIfMissing(db, "sessions", column, `TEXT DEFAULT ''`);
|
|
790
780
|
}
|
|
781
|
+
addColumnIfMissing(db, "model_pricing", "cache_write_1h_per_1m", "REAL NOT NULL DEFAULT 0");
|
|
782
|
+
addColumnIfMissing(db, "model_pricing", "cache_storage_per_1m_hour", "REAL NOT NULL DEFAULT 0");
|
|
791
783
|
db.exec(`
|
|
792
784
|
CREATE INDEX IF NOT EXISTS idx_requests_machine ON requests(machine_id);
|
|
793
785
|
CREATE INDEX IF NOT EXISTS idx_sessions_machine ON sessions(machine_id);
|
|
@@ -948,17 +940,22 @@ function querySummary(db, period, machine, allMachines = false) {
|
|
|
948
940
|
const codexTotals = db.prepare(`
|
|
949
941
|
SELECT COALESCE(SUM(total_cost_usd), 0) as cost_usd,
|
|
950
942
|
COALESCE(SUM(total_tokens), 0) as tokens,
|
|
943
|
+
COALESCE(SUM(request_count), 0) as requests,
|
|
951
944
|
COUNT(*) as sessions
|
|
952
945
|
FROM sessions
|
|
953
946
|
WHERE ${sWhere}${machineClause}
|
|
954
947
|
AND id NOT IN (SELECT DISTINCT session_id FROM requests)
|
|
955
948
|
`).get();
|
|
956
|
-
const
|
|
949
|
+
const requestSessionCount = db.prepare(`
|
|
950
|
+
SELECT COUNT(DISTINCT session_id) as sessions
|
|
951
|
+
FROM requests
|
|
952
|
+
WHERE ${rWhere}${machineClause}
|
|
953
|
+
`).get();
|
|
957
954
|
return {
|
|
958
955
|
total_usd: r.total_usd + codexTotals.cost_usd,
|
|
959
|
-
requests: r.requests,
|
|
956
|
+
requests: r.requests + codexTotals.requests,
|
|
960
957
|
tokens: r.tokens + codexTotals.tokens,
|
|
961
|
-
sessions:
|
|
958
|
+
sessions: requestSessionCount.sessions + codexTotals.sessions,
|
|
962
959
|
period
|
|
963
960
|
};
|
|
964
961
|
}
|
|
@@ -1037,9 +1034,7 @@ function queryAgentBreakdown(db, period = "all") {
|
|
|
1037
1034
|
}
|
|
1038
1035
|
return [...groups.values()].sort((a, b) => b.api_equivalent_usd - a.api_equivalent_usd);
|
|
1039
1036
|
}
|
|
1040
|
-
function
|
|
1041
|
-
if (projectName && projectName.trim() !== "")
|
|
1042
|
-
return projectName;
|
|
1037
|
+
function pathProjectLabel(projectPath) {
|
|
1043
1038
|
if (!projectPath)
|
|
1044
1039
|
return "";
|
|
1045
1040
|
const segments = projectPath.split("/").filter(Boolean);
|
|
@@ -1048,12 +1043,45 @@ function labelForPath(projectPath, projectName) {
|
|
|
1048
1043
|
if (projectPrefix.test(seg))
|
|
1049
1044
|
return seg;
|
|
1050
1045
|
}
|
|
1051
|
-
const generic = new Set([
|
|
1046
|
+
const generic = new Set([
|
|
1047
|
+
"web",
|
|
1048
|
+
"app",
|
|
1049
|
+
"apps",
|
|
1050
|
+
"packages",
|
|
1051
|
+
"src",
|
|
1052
|
+
"lib",
|
|
1053
|
+
"server",
|
|
1054
|
+
"client",
|
|
1055
|
+
"api",
|
|
1056
|
+
"frontend",
|
|
1057
|
+
"backend",
|
|
1058
|
+
"home",
|
|
1059
|
+
"users",
|
|
1060
|
+
"workspace",
|
|
1061
|
+
"workspaces",
|
|
1062
|
+
"hasna"
|
|
1063
|
+
]);
|
|
1052
1064
|
for (let i = segments.length - 1;i >= 0; i--) {
|
|
1053
1065
|
if (!generic.has(segments[i].toLowerCase()))
|
|
1054
1066
|
return segments[i];
|
|
1055
1067
|
}
|
|
1056
|
-
return
|
|
1068
|
+
return null;
|
|
1069
|
+
}
|
|
1070
|
+
function isRepoLikeLabel(label) {
|
|
1071
|
+
return /^(open|skill|hook|service|connect|platform|agent|tool|iapp|project|scaffold|capp)-/.test(label) || label.includes("-");
|
|
1072
|
+
}
|
|
1073
|
+
function labelForPath(projectPath, projectName) {
|
|
1074
|
+
const pathLabel = pathProjectLabel(projectPath);
|
|
1075
|
+
if (pathLabel && (!projectName || projectName.trim() === "" || isRepoLikeLabel(pathLabel)))
|
|
1076
|
+
return pathLabel;
|
|
1077
|
+
if (projectName && projectName.trim() !== "")
|
|
1078
|
+
return projectName;
|
|
1079
|
+
if (pathLabel)
|
|
1080
|
+
return pathLabel;
|
|
1081
|
+
return projectPath;
|
|
1082
|
+
}
|
|
1083
|
+
function groupKeyForPath(projectPath, projectName) {
|
|
1084
|
+
return labelForPath(projectPath, projectName).trim().toLowerCase();
|
|
1057
1085
|
}
|
|
1058
1086
|
function queryProjectBreakdown(db, period = "all") {
|
|
1059
1087
|
const requestWhere = requestPeriodWhere(period);
|
|
@@ -1068,14 +1096,15 @@ function queryProjectBreakdown(db, period = "all") {
|
|
|
1068
1096
|
const label = labelForPath(s.project_path, s.project_name);
|
|
1069
1097
|
if (!label)
|
|
1070
1098
|
continue;
|
|
1071
|
-
const
|
|
1099
|
+
const key = groupKeyForPath(s.project_path, s.project_name);
|
|
1100
|
+
const g = groups.get(key) ?? { label, sessionIds: [], samplePath: s.project_path };
|
|
1072
1101
|
g.sessionIds.push(s.id);
|
|
1073
1102
|
if (!g.samplePath)
|
|
1074
1103
|
g.samplePath = s.project_path;
|
|
1075
|
-
groups.set(
|
|
1104
|
+
groups.set(key, g);
|
|
1076
1105
|
}
|
|
1077
1106
|
const result = [];
|
|
1078
|
-
for (const
|
|
1107
|
+
for (const g of groups.values()) {
|
|
1079
1108
|
const placeholders = g.sessionIds.map(() => "?").join(",");
|
|
1080
1109
|
const reqStats = placeholders.length ? db.prepare(`
|
|
1081
1110
|
SELECT
|
|
@@ -1106,7 +1135,7 @@ function queryProjectBreakdown(db, period = "all") {
|
|
|
1106
1135
|
const lastActive = [reqStats.last_active, sessionOnlyStats.last_active].filter(Boolean).sort().at(-1) ?? "";
|
|
1107
1136
|
result.push({
|
|
1108
1137
|
project_path: g.samplePath,
|
|
1109
|
-
project_name: label,
|
|
1138
|
+
project_name: g.label,
|
|
1110
1139
|
sessions: totalSessions,
|
|
1111
1140
|
requests: reqStats.requests + sessionOnlyStats.requests,
|
|
1112
1141
|
total_tokens: reqStats.total_tokens + sessionOnlyStats.total_tokens,
|
|
@@ -1346,17 +1375,48 @@ function queryBillingSummary(db, period) {
|
|
|
1346
1375
|
}
|
|
1347
1376
|
return { total_usd: total, by_provider };
|
|
1348
1377
|
}
|
|
1349
|
-
function listMachines(db) {
|
|
1378
|
+
function listMachines(db, period = "all") {
|
|
1379
|
+
const rWhere = requestPeriodWhere(period);
|
|
1380
|
+
const sWhere = sessionPeriodWhere(period);
|
|
1350
1381
|
return db.prepare(`
|
|
1382
|
+
WITH request_stats AS (
|
|
1383
|
+
SELECT
|
|
1384
|
+
machine_id,
|
|
1385
|
+
COUNT(DISTINCT session_id) as sessions,
|
|
1386
|
+
COUNT(*) as requests,
|
|
1387
|
+
COALESCE(SUM(cost_usd), 0) as total_cost_usd,
|
|
1388
|
+
MAX(timestamp) as last_active
|
|
1389
|
+
FROM requests
|
|
1390
|
+
WHERE machine_id != ''
|
|
1391
|
+
AND ${rWhere}
|
|
1392
|
+
GROUP BY machine_id
|
|
1393
|
+
),
|
|
1394
|
+
session_only_stats AS (
|
|
1395
|
+
SELECT
|
|
1396
|
+
machine_id,
|
|
1397
|
+
COUNT(*) as sessions,
|
|
1398
|
+
COALESCE(SUM(request_count), 0) as requests,
|
|
1399
|
+
COALESCE(SUM(total_cost_usd), 0) as total_cost_usd,
|
|
1400
|
+
MAX(started_at) as last_active
|
|
1401
|
+
FROM sessions
|
|
1402
|
+
WHERE machine_id != ''
|
|
1403
|
+
AND ${sWhere}
|
|
1404
|
+
AND id NOT IN (SELECT DISTINCT session_id FROM requests)
|
|
1405
|
+
GROUP BY machine_id
|
|
1406
|
+
),
|
|
1407
|
+
combined AS (
|
|
1408
|
+
SELECT * FROM request_stats
|
|
1409
|
+
UNION ALL
|
|
1410
|
+
SELECT * FROM session_only_stats
|
|
1411
|
+
)
|
|
1351
1412
|
SELECT
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
COALESCE((
|
|
1355
|
-
COALESCE(SUM(
|
|
1356
|
-
MAX(
|
|
1357
|
-
FROM
|
|
1358
|
-
|
|
1359
|
-
GROUP BY s.machine_id
|
|
1413
|
+
machine_id,
|
|
1414
|
+
COALESCE(SUM(sessions), 0) as sessions,
|
|
1415
|
+
COALESCE(SUM(requests), 0) as requests,
|
|
1416
|
+
COALESCE(SUM(total_cost_usd), 0) as total_cost_usd,
|
|
1417
|
+
MAX(last_active) as last_active
|
|
1418
|
+
FROM combined
|
|
1419
|
+
GROUP BY machine_id
|
|
1360
1420
|
ORDER BY total_cost_usd DESC
|
|
1361
1421
|
`).all();
|
|
1362
1422
|
}
|
|
@@ -1438,17 +1498,21 @@ function listMachineRegistry(db) {
|
|
|
1438
1498
|
}
|
|
1439
1499
|
function dedupeRequests(db) {
|
|
1440
1500
|
const dupes = db.prepare(`
|
|
1441
|
-
SELECT source_request_id, agent, MIN(id) as keep_id, COUNT(*) as cnt
|
|
1501
|
+
SELECT source_request_id, agent, COALESCE(machine_id, '') as machine_id, MIN(id) as keep_id, COUNT(*) as cnt
|
|
1442
1502
|
FROM requests
|
|
1443
1503
|
WHERE source_request_id != '' AND source_request_id IS NOT NULL
|
|
1444
|
-
GROUP BY source_request_id, agent
|
|
1504
|
+
GROUP BY source_request_id, agent, COALESCE(machine_id, '')
|
|
1445
1505
|
HAVING cnt > 1
|
|
1446
1506
|
`).all();
|
|
1447
1507
|
let removed = 0;
|
|
1448
1508
|
for (const row of dupes) {
|
|
1449
1509
|
const result = db.prepare(`
|
|
1450
|
-
DELETE FROM requests
|
|
1451
|
-
|
|
1510
|
+
DELETE FROM requests
|
|
1511
|
+
WHERE source_request_id = ?
|
|
1512
|
+
AND agent = ?
|
|
1513
|
+
AND COALESCE(machine_id, '') = ?
|
|
1514
|
+
AND id != ?
|
|
1515
|
+
`).run(row.source_request_id, row.agent, row.machine_id, row.keep_id);
|
|
1452
1516
|
removed += result.changes;
|
|
1453
1517
|
}
|
|
1454
1518
|
return removed;
|
|
@@ -1731,10 +1795,303 @@ async function syncOpenProjectsRegistry(db, listActiveProjects) {
|
|
|
1731
1795
|
}
|
|
1732
1796
|
return { imported, skipped };
|
|
1733
1797
|
}
|
|
1798
|
+
// src/lib/peer-sync.ts
|
|
1799
|
+
init_database();
|
|
1800
|
+
import { Database as BunDatabase } from "bun:sqlite";
|
|
1801
|
+
import { existsSync as existsSync3 } from "fs";
|
|
1802
|
+
|
|
1803
|
+
// src/lib/package-metadata.ts
|
|
1804
|
+
import { readFileSync as readFileSync2 } from "fs";
|
|
1805
|
+
var cachedMetadata = null;
|
|
1806
|
+
function getPackageMetadata() {
|
|
1807
|
+
if (cachedMetadata)
|
|
1808
|
+
return cachedMetadata;
|
|
1809
|
+
const raw = readFileSync2(new URL("../../package.json", import.meta.url), "utf8");
|
|
1810
|
+
const parsed = JSON.parse(raw);
|
|
1811
|
+
cachedMetadata = {
|
|
1812
|
+
name: parsed.name ?? "@hasna/economy",
|
|
1813
|
+
version: parsed.version ?? "0.0.0"
|
|
1814
|
+
};
|
|
1815
|
+
return cachedMetadata;
|
|
1816
|
+
}
|
|
1817
|
+
var packageMetadata = getPackageMetadata();
|
|
1818
|
+
|
|
1819
|
+
// src/lib/peer-sync.ts
|
|
1820
|
+
var GENERIC_PEER_TABLES = [
|
|
1821
|
+
"usage_snapshots",
|
|
1822
|
+
"subscriptions",
|
|
1823
|
+
"billing_daily",
|
|
1824
|
+
"savings_daily",
|
|
1825
|
+
"budgets",
|
|
1826
|
+
"goals",
|
|
1827
|
+
"model_pricing",
|
|
1828
|
+
"machines"
|
|
1829
|
+
];
|
|
1830
|
+
function quoteIdent(identifier) {
|
|
1831
|
+
return `"${identifier.replace(/"/g, '""')}"`;
|
|
1832
|
+
}
|
|
1833
|
+
function tableExists(db, table) {
|
|
1834
|
+
const row = db.prepare(`SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?`).get(table);
|
|
1835
|
+
return Boolean(row);
|
|
1836
|
+
}
|
|
1837
|
+
function tableColumns(db, table) {
|
|
1838
|
+
if (!tableExists(db, table))
|
|
1839
|
+
return [];
|
|
1840
|
+
return db.prepare(`PRAGMA table_info(${quoteIdent(table)})`).all();
|
|
1841
|
+
}
|
|
1842
|
+
function commonColumns(source, target, table) {
|
|
1843
|
+
const sourceCols = new Set(tableColumns(source, table).map((c) => c.name));
|
|
1844
|
+
return tableColumns(target, table).map((c) => c.name).filter((c) => sourceCols.has(c));
|
|
1845
|
+
}
|
|
1846
|
+
function primaryKeyColumns(db, table) {
|
|
1847
|
+
return tableColumns(db, table).filter((c) => c.pk > 0).sort((a, b) => a.pk - b.pk).map((c) => c.name);
|
|
1848
|
+
}
|
|
1849
|
+
function selectRows(source, table, columns) {
|
|
1850
|
+
if (columns.length === 0)
|
|
1851
|
+
return [];
|
|
1852
|
+
const select = columns.map(quoteIdent).join(", ");
|
|
1853
|
+
return source.prepare(`SELECT ${select} FROM ${quoteIdent(table)}`).all();
|
|
1854
|
+
}
|
|
1855
|
+
function rowByKey(target, table, keyColumns, row) {
|
|
1856
|
+
if (keyColumns.length === 0)
|
|
1857
|
+
return null;
|
|
1858
|
+
if (keyColumns.some((c) => row[c] == null))
|
|
1859
|
+
return null;
|
|
1860
|
+
const where = keyColumns.map((c) => `${quoteIdent(c)} = ?`).join(" AND ");
|
|
1861
|
+
return target.prepare(`SELECT * FROM ${quoteIdent(table)} WHERE ${where}`).get(...keyColumns.map((c) => row[c]));
|
|
1862
|
+
}
|
|
1863
|
+
function hasId(target, table, id) {
|
|
1864
|
+
return target.prepare(`SELECT id, machine_id FROM ${quoteIdent(table)} WHERE id = ?`).get(id);
|
|
1865
|
+
}
|
|
1866
|
+
function shouldReplace(source, existing) {
|
|
1867
|
+
if (!existing)
|
|
1868
|
+
return true;
|
|
1869
|
+
const sourceUpdated = source["updated_at"];
|
|
1870
|
+
const existingUpdated = existing["updated_at"];
|
|
1871
|
+
if (typeof sourceUpdated === "string" && typeof existingUpdated === "string" && existingUpdated !== "") {
|
|
1872
|
+
return sourceUpdated >= existingUpdated;
|
|
1873
|
+
}
|
|
1874
|
+
return true;
|
|
1875
|
+
}
|
|
1876
|
+
function normalizeRow(row, columns, sourceMachine, now) {
|
|
1877
|
+
const next = { ...row };
|
|
1878
|
+
if (columns.includes("machine_id") && (!next["machine_id"] || next["machine_id"] === "")) {
|
|
1879
|
+
next["machine_id"] = sourceMachine;
|
|
1880
|
+
}
|
|
1881
|
+
if (columns.includes("updated_at") && (!next["updated_at"] || next["updated_at"] === "")) {
|
|
1882
|
+
next["updated_at"] = next["timestamp"] ?? next["started_at"] ?? next["created_at"] ?? now;
|
|
1883
|
+
}
|
|
1884
|
+
if (columns.includes("synced_at") && next["synced_at"] == null)
|
|
1885
|
+
next["synced_at"] = "";
|
|
1886
|
+
if (columns.includes("attribution_tag") && next["attribution_tag"] == null)
|
|
1887
|
+
next["attribution_tag"] = "";
|
|
1888
|
+
return next;
|
|
1889
|
+
}
|
|
1890
|
+
function insertOrReplace(target, table, columns, row) {
|
|
1891
|
+
const colSql = columns.map(quoteIdent).join(", ");
|
|
1892
|
+
const placeholders = columns.map(() => "?").join(", ");
|
|
1893
|
+
target.prepare(`
|
|
1894
|
+
INSERT OR REPLACE INTO ${quoteIdent(table)} (${colSql})
|
|
1895
|
+
VALUES (${placeholders})
|
|
1896
|
+
`).run(...columns.map((c) => row[c] ?? null));
|
|
1897
|
+
}
|
|
1898
|
+
function collisionId(target, table, machine, originalId) {
|
|
1899
|
+
const base = `${machine || "peer"}:${originalId}`;
|
|
1900
|
+
const baseRow = hasId(target, table, base);
|
|
1901
|
+
if (!baseRow || String(baseRow["machine_id"] ?? "") === machine)
|
|
1902
|
+
return base;
|
|
1903
|
+
for (let i = 2;; i++) {
|
|
1904
|
+
const candidate = `${base}:${i}`;
|
|
1905
|
+
const row = hasId(target, table, candidate);
|
|
1906
|
+
if (!row || String(row["machine_id"] ?? "") === machine)
|
|
1907
|
+
return candidate;
|
|
1908
|
+
}
|
|
1909
|
+
}
|
|
1910
|
+
function mergeIdentityTable(target, source, table, sourceMachine, now, sessionIdMap) {
|
|
1911
|
+
const stats = { table, inserted: 0, updated: 0, skipped: 0, collisions: 0 };
|
|
1912
|
+
const columns = commonColumns(source, target, table);
|
|
1913
|
+
const rows = selectRows(source, table, columns);
|
|
1914
|
+
const idMap = new Map;
|
|
1915
|
+
for (const raw of rows) {
|
|
1916
|
+
const row = normalizeRow(raw, columns, sourceMachine, now);
|
|
1917
|
+
const originalId = String(row["id"] ?? "");
|
|
1918
|
+
if (!originalId) {
|
|
1919
|
+
stats.skipped++;
|
|
1920
|
+
continue;
|
|
1921
|
+
}
|
|
1922
|
+
const machine = String(row["machine_id"] ?? "");
|
|
1923
|
+
const directExisting = hasId(target, table, originalId);
|
|
1924
|
+
if (directExisting && String(directExisting["machine_id"] ?? "") !== machine) {
|
|
1925
|
+
row["id"] = collisionId(target, table, machine, originalId);
|
|
1926
|
+
stats.collisions++;
|
|
1927
|
+
}
|
|
1928
|
+
if (table === "requests" && sessionIdMap) {
|
|
1929
|
+
const originalSessionId = String(row["session_id"] ?? "");
|
|
1930
|
+
row["session_id"] = sessionIdMap.get(originalSessionId) ?? originalSessionId;
|
|
1931
|
+
}
|
|
1932
|
+
const existing = hasId(target, table, String(row["id"]));
|
|
1933
|
+
idMap.set(originalId, String(row["id"]));
|
|
1934
|
+
if (existing && !shouldReplace(row, existing)) {
|
|
1935
|
+
stats.skipped++;
|
|
1936
|
+
continue;
|
|
1937
|
+
}
|
|
1938
|
+
insertOrReplace(target, table, columns, row);
|
|
1939
|
+
if (existing)
|
|
1940
|
+
stats.updated++;
|
|
1941
|
+
else
|
|
1942
|
+
stats.inserted++;
|
|
1943
|
+
}
|
|
1944
|
+
return { stats, idMap };
|
|
1945
|
+
}
|
|
1946
|
+
function mergeProjects(target, source) {
|
|
1947
|
+
const table = "projects";
|
|
1948
|
+
const stats = { table, inserted: 0, updated: 0, skipped: 0, collisions: 0 };
|
|
1949
|
+
const columns = commonColumns(source, target, table);
|
|
1950
|
+
const rows = selectRows(source, table, columns);
|
|
1951
|
+
for (const raw of rows) {
|
|
1952
|
+
const row = { ...raw };
|
|
1953
|
+
const path = String(row["path"] ?? "");
|
|
1954
|
+
const id = String(row["id"] ?? "");
|
|
1955
|
+
if (!path || !id) {
|
|
1956
|
+
stats.skipped++;
|
|
1957
|
+
continue;
|
|
1958
|
+
}
|
|
1959
|
+
const existingByPath = target.prepare(`SELECT * FROM projects WHERE path = ?`).get(path);
|
|
1960
|
+
if (existingByPath) {
|
|
1961
|
+
row["id"] = existingByPath["id"] ?? id;
|
|
1962
|
+
insertOrReplace(target, table, columns, row);
|
|
1963
|
+
stats.updated++;
|
|
1964
|
+
continue;
|
|
1965
|
+
}
|
|
1966
|
+
const existingById = target.prepare(`SELECT * FROM projects WHERE id = ?`).get(id);
|
|
1967
|
+
if (existingById && String(existingById["path"] ?? "") !== path) {
|
|
1968
|
+
row["id"] = `peer:${id}`;
|
|
1969
|
+
stats.collisions++;
|
|
1970
|
+
while (target.prepare(`SELECT id FROM projects WHERE id = ?`).get(row["id"])) {
|
|
1971
|
+
row["id"] = `peer:${String(row["id"])}`;
|
|
1972
|
+
}
|
|
1973
|
+
}
|
|
1974
|
+
insertOrReplace(target, table, columns, row);
|
|
1975
|
+
stats.inserted++;
|
|
1976
|
+
}
|
|
1977
|
+
return stats;
|
|
1978
|
+
}
|
|
1979
|
+
function mergeGenericTable(target, source, table, sourceMachine, now) {
|
|
1980
|
+
const stats = { table, inserted: 0, updated: 0, skipped: 0, collisions: 0 };
|
|
1981
|
+
const columns = commonColumns(source, target, table);
|
|
1982
|
+
const keyColumns = primaryKeyColumns(target, table).filter((c) => columns.includes(c));
|
|
1983
|
+
const rows = selectRows(source, table, columns);
|
|
1984
|
+
for (const raw of rows) {
|
|
1985
|
+
const row = normalizeRow(raw, columns, sourceMachine, now);
|
|
1986
|
+
const existing = rowByKey(target, table, keyColumns, row);
|
|
1987
|
+
if (existing && !shouldReplace(row, existing)) {
|
|
1988
|
+
stats.skipped++;
|
|
1989
|
+
continue;
|
|
1990
|
+
}
|
|
1991
|
+
insertOrReplace(target, table, columns, row);
|
|
1992
|
+
if (existing)
|
|
1993
|
+
stats.updated++;
|
|
1994
|
+
else
|
|
1995
|
+
stats.inserted++;
|
|
1996
|
+
}
|
|
1997
|
+
return stats;
|
|
1998
|
+
}
|
|
1999
|
+
function detectSourceMachine(source, fallback) {
|
|
2000
|
+
if (fallback && fallback.trim())
|
|
2001
|
+
return fallback.trim();
|
|
2002
|
+
const counts = new Map;
|
|
2003
|
+
for (const table of ["sessions", "requests", "usage_snapshots"]) {
|
|
2004
|
+
if (!tableExists(source, table))
|
|
2005
|
+
continue;
|
|
2006
|
+
const rows = source.prepare(`
|
|
2007
|
+
SELECT machine_id, COUNT(*) as cnt
|
|
2008
|
+
FROM ${quoteIdent(table)}
|
|
2009
|
+
WHERE machine_id != '' AND machine_id IS NOT NULL
|
|
2010
|
+
GROUP BY machine_id
|
|
2011
|
+
`).all();
|
|
2012
|
+
for (const row of rows) {
|
|
2013
|
+
counts.set(row.machine_id, (counts.get(row.machine_id) ?? 0) + row.cnt);
|
|
2014
|
+
}
|
|
2015
|
+
}
|
|
2016
|
+
let best = "";
|
|
2017
|
+
let bestCount = -1;
|
|
2018
|
+
for (const [machine, count] of counts.entries()) {
|
|
2019
|
+
if (count > bestCount) {
|
|
2020
|
+
best = machine;
|
|
2021
|
+
bestCount = count;
|
|
2022
|
+
}
|
|
2023
|
+
}
|
|
2024
|
+
return best || "peer";
|
|
2025
|
+
}
|
|
2026
|
+
function ensureMachineRegistry(target, machine, now) {
|
|
2027
|
+
if (!machine)
|
|
2028
|
+
return;
|
|
2029
|
+
target.prepare(`
|
|
2030
|
+
INSERT INTO machines (machine_id, hostname, last_seen_at, last_push_at, last_pull_at, economy_version, updated_at)
|
|
2031
|
+
VALUES (?, ?, ?, NULL, ?, ?, ?)
|
|
2032
|
+
ON CONFLICT(machine_id) DO UPDATE SET
|
|
2033
|
+
hostname = COALESCE(NULLIF(machines.hostname, ''), excluded.hostname),
|
|
2034
|
+
last_seen_at = CASE
|
|
2035
|
+
WHEN machines.last_seen_at IS NULL OR machines.last_seen_at < excluded.last_seen_at THEN excluded.last_seen_at
|
|
2036
|
+
ELSE machines.last_seen_at
|
|
2037
|
+
END,
|
|
2038
|
+
last_pull_at = excluded.last_pull_at,
|
|
2039
|
+
economy_version = excluded.economy_version,
|
|
2040
|
+
updated_at = excluded.updated_at
|
|
2041
|
+
`).run(machine, machine, now, now, packageMetadata.version, now);
|
|
2042
|
+
}
|
|
2043
|
+
function openSourceDatabase(path) {
|
|
2044
|
+
try {
|
|
2045
|
+
return new BunDatabase(path, { readonly: true });
|
|
2046
|
+
} catch {
|
|
2047
|
+
return new BunDatabase(path);
|
|
2048
|
+
}
|
|
2049
|
+
}
|
|
2050
|
+
function mergePeerDatabase(target, sourcePath, opts = {}) {
|
|
2051
|
+
if (!existsSync3(sourcePath))
|
|
2052
|
+
throw new Error(`source database does not exist: ${sourcePath}`);
|
|
2053
|
+
const source = openSourceDatabase(sourcePath);
|
|
2054
|
+
const now = opts.now ?? new Date().toISOString();
|
|
2055
|
+
const sourceMachine = detectSourceMachine(source, opts.sourceMachine);
|
|
2056
|
+
const tables = [];
|
|
2057
|
+
try {
|
|
2058
|
+
target.exec("PRAGMA foreign_keys = OFF");
|
|
2059
|
+
target.exec("BEGIN IMMEDIATE");
|
|
2060
|
+
try {
|
|
2061
|
+
tables.push(mergeProjects(target, source));
|
|
2062
|
+
const sessionMerge = mergeIdentityTable(target, source, "sessions", sourceMachine, now);
|
|
2063
|
+
tables.push(sessionMerge.stats);
|
|
2064
|
+
tables.push(mergeIdentityTable(target, source, "requests", sourceMachine, now, sessionMerge.idMap).stats);
|
|
2065
|
+
for (const table of GENERIC_PEER_TABLES) {
|
|
2066
|
+
tables.push(mergeGenericTable(target, source, table, sourceMachine, now));
|
|
2067
|
+
}
|
|
2068
|
+
ensureMachineRegistry(target, sourceMachine, now);
|
|
2069
|
+
target.exec("COMMIT");
|
|
2070
|
+
} catch (err) {
|
|
2071
|
+
target.exec("ROLLBACK");
|
|
2072
|
+
throw err;
|
|
2073
|
+
} finally {
|
|
2074
|
+
target.exec("PRAGMA foreign_keys = ON");
|
|
2075
|
+
}
|
|
2076
|
+
} finally {
|
|
2077
|
+
source.close();
|
|
2078
|
+
}
|
|
2079
|
+
const deduped = dedupeRequests(target);
|
|
2080
|
+
const rowsWritten = tables.reduce((sum, table) => sum + table.inserted + table.updated, 0);
|
|
2081
|
+
const collisions = tables.reduce((sum, table) => sum + table.collisions, 0);
|
|
2082
|
+
return {
|
|
2083
|
+
source_path: sourcePath,
|
|
2084
|
+
source_machine: sourceMachine,
|
|
2085
|
+
rows_written: rowsWritten,
|
|
2086
|
+
collisions,
|
|
2087
|
+
deduped,
|
|
2088
|
+
tables: tables.filter((t) => t.inserted || t.updated || t.skipped || t.collisions)
|
|
2089
|
+
};
|
|
2090
|
+
}
|
|
1734
2091
|
// src/ingest/claude.ts
|
|
1735
2092
|
init_database();
|
|
1736
2093
|
init_pricing();
|
|
1737
|
-
import { readdirSync as readdirSync2, readFileSync as
|
|
2094
|
+
import { readdirSync as readdirSync2, readFileSync as readFileSync3, existsSync as existsSync4, statSync as statSync2 } from "fs";
|
|
1738
2095
|
import { homedir as homedir2 } from "os";
|
|
1739
2096
|
import { join as join3, basename } from "path";
|
|
1740
2097
|
|
|
@@ -1903,7 +2260,7 @@ async function ingestTakumi(db, verbose = false, projectsDir = TAKUMI_PROJECTS_D
|
|
|
1903
2260
|
return ingestJsonlProjects(db, projectsDir, "takumi", verbose);
|
|
1904
2261
|
}
|
|
1905
2262
|
async function ingestJsonlProjects(db, projectsDir, agentName, verbose = false) {
|
|
1906
|
-
if (!
|
|
2263
|
+
if (!existsSync4(projectsDir)) {
|
|
1907
2264
|
if (verbose)
|
|
1908
2265
|
console.log(`${agentName} projects dir not found:`, projectsDir);
|
|
1909
2266
|
return { files: 0, requests: 0, sessions: 0 };
|
|
@@ -1932,7 +2289,7 @@ async function ingestJsonlProjects(db, projectsDir, agentName, verbose = false)
|
|
|
1932
2289
|
continue;
|
|
1933
2290
|
let lines;
|
|
1934
2291
|
try {
|
|
1935
|
-
lines =
|
|
2292
|
+
lines = readFileSync3(filePath, "utf-8").split(`
|
|
1936
2293
|
`).filter((l) => l.trim());
|
|
1937
2294
|
} catch {
|
|
1938
2295
|
continue;
|
|
@@ -2051,10 +2408,10 @@ function supportsClaudeDataResidencyPricing(model) {
|
|
|
2051
2408
|
// src/ingest/codex.ts
|
|
2052
2409
|
init_database();
|
|
2053
2410
|
init_pricing();
|
|
2054
|
-
import { existsSync as
|
|
2411
|
+
import { existsSync as existsSync5, readFileSync as readFileSync4 } from "fs";
|
|
2055
2412
|
import { homedir as homedir3 } from "os";
|
|
2056
2413
|
import { join as join4, basename as basename2 } from "path";
|
|
2057
|
-
import { Database as
|
|
2414
|
+
import { Database as BunDatabase2 } from "bun:sqlite";
|
|
2058
2415
|
var DEFAULT_CODEX_DB_PATH = join4(homedir3(), ".codex", "state_5.sqlite");
|
|
2059
2416
|
var DEFAULT_CODEX_CONFIG_PATH = join4(homedir3(), ".codex", "config.toml");
|
|
2060
2417
|
var CODEX_INGEST_VERSION = "rollout-aggregate-v3";
|
|
@@ -2066,10 +2423,10 @@ function codexConfigPath() {
|
|
|
2066
2423
|
}
|
|
2067
2424
|
function readCodexModel() {
|
|
2068
2425
|
const configPath = codexConfigPath();
|
|
2069
|
-
if (!
|
|
2426
|
+
if (!existsSync5(configPath))
|
|
2070
2427
|
return "gpt-5-codex";
|
|
2071
2428
|
try {
|
|
2072
|
-
const content =
|
|
2429
|
+
const content = readFileSync4(configPath, "utf-8");
|
|
2073
2430
|
const match = content.match(/^model\s*=\s*"([^"]+)"/m);
|
|
2074
2431
|
return match?.[1] ?? "gpt-5-codex";
|
|
2075
2432
|
} catch {
|
|
@@ -2092,7 +2449,7 @@ function openCodexDb(dbPath, verbose) {
|
|
|
2092
2449
|
for (const readonly of [true, false]) {
|
|
2093
2450
|
let codexDb = null;
|
|
2094
2451
|
try {
|
|
2095
|
-
codexDb = readonly ? new
|
|
2452
|
+
codexDb = readonly ? new BunDatabase2(dbPath, { readonly: true }) : new BunDatabase2(dbPath);
|
|
2096
2453
|
codexDb.prepare("PRAGMA schema_version").get();
|
|
2097
2454
|
return codexDb;
|
|
2098
2455
|
} catch (error) {
|
|
@@ -2107,12 +2464,12 @@ function openCodexDb(dbPath, verbose) {
|
|
|
2107
2464
|
return null;
|
|
2108
2465
|
}
|
|
2109
2466
|
function readTokenEvents(rolloutPath) {
|
|
2110
|
-
if (!rolloutPath || !
|
|
2467
|
+
if (!rolloutPath || !existsSync5(rolloutPath))
|
|
2111
2468
|
return [];
|
|
2112
2469
|
const fallbackUsages = new Map;
|
|
2113
2470
|
let fallbackTimestamp;
|
|
2114
2471
|
let aggregate = null;
|
|
2115
|
-
for (const line of
|
|
2472
|
+
for (const line of readFileSync4(rolloutPath, "utf-8").split(`
|
|
2116
2473
|
`)) {
|
|
2117
2474
|
if (!line.trim())
|
|
2118
2475
|
continue;
|
|
@@ -2184,7 +2541,7 @@ function fallbackEvents(totalTokens) {
|
|
|
2184
2541
|
}
|
|
2185
2542
|
async function ingestCodex(db, verbose = false) {
|
|
2186
2543
|
const dbPath = codexDbPath();
|
|
2187
|
-
if (!
|
|
2544
|
+
if (!existsSync5(dbPath)) {
|
|
2188
2545
|
if (verbose)
|
|
2189
2546
|
console.log("Codex DB not found:", dbPath);
|
|
2190
2547
|
return { sessions: 0, requests: 0 };
|
|
@@ -2267,7 +2624,7 @@ async function ingestCodex(db, verbose = false) {
|
|
|
2267
2624
|
// src/ingest/gemini.ts
|
|
2268
2625
|
init_database();
|
|
2269
2626
|
init_pricing();
|
|
2270
|
-
import { readdirSync as readdirSync3, readFileSync as
|
|
2627
|
+
import { readdirSync as readdirSync3, readFileSync as readFileSync5, existsSync as existsSync6, statSync as statSync3 } from "fs";
|
|
2271
2628
|
import { homedir as homedir4 } from "os";
|
|
2272
2629
|
import { join as join5, basename as basename3 } from "path";
|
|
2273
2630
|
var DEFAULT_GEMINI_TMP_DIR = join5(homedir4(), ".gemini", "tmp");
|
|
@@ -2288,7 +2645,7 @@ function numberField(...values) {
|
|
|
2288
2645
|
function listProjectDirs(...roots) {
|
|
2289
2646
|
const dirs = new Set;
|
|
2290
2647
|
for (const root of roots) {
|
|
2291
|
-
if (!
|
|
2648
|
+
if (!existsSync6(root))
|
|
2292
2649
|
continue;
|
|
2293
2650
|
try {
|
|
2294
2651
|
for (const entry of readdirSync3(root, { withFileTypes: true })) {
|
|
@@ -2306,15 +2663,15 @@ function projectRoot(projectDir, chatData) {
|
|
|
2306
2663
|
return chatData.project_path;
|
|
2307
2664
|
const rootFile = join5(projectDir, ".project_root");
|
|
2308
2665
|
try {
|
|
2309
|
-
if (
|
|
2310
|
-
return
|
|
2666
|
+
if (existsSync6(rootFile))
|
|
2667
|
+
return readFileSync5(rootFile, "utf-8").trim();
|
|
2311
2668
|
} catch {}
|
|
2312
2669
|
return "";
|
|
2313
2670
|
}
|
|
2314
2671
|
async function ingestGemini(db, verbose) {
|
|
2315
2672
|
const tmpDir = geminiTmpDir();
|
|
2316
2673
|
const historyDir = geminiHistoryDir();
|
|
2317
|
-
if (!
|
|
2674
|
+
if (!existsSync6(tmpDir) && !existsSync6(historyDir)) {
|
|
2318
2675
|
if (verbose)
|
|
2319
2676
|
console.log("Gemini tmp/history dirs not found:", tmpDir, historyDir);
|
|
2320
2677
|
return { sessions: 0, requests: 0 };
|
|
@@ -2327,7 +2684,7 @@ async function ingestGemini(db, verbose) {
|
|
|
2327
2684
|
const projectDirs = listProjectDirs(tmpDir, historyDir);
|
|
2328
2685
|
for (const projectDir of projectDirs) {
|
|
2329
2686
|
const chatsDir = join5(projectDir, "chats");
|
|
2330
|
-
if (!
|
|
2687
|
+
if (!existsSync6(chatsDir))
|
|
2331
2688
|
continue;
|
|
2332
2689
|
let chatFiles = [];
|
|
2333
2690
|
try {
|
|
@@ -2348,7 +2705,7 @@ async function ingestGemini(db, verbose) {
|
|
|
2348
2705
|
continue;
|
|
2349
2706
|
let chatData;
|
|
2350
2707
|
try {
|
|
2351
|
-
chatData = JSON.parse(
|
|
2708
|
+
chatData = JSON.parse(readFileSync5(filePath, "utf-8"));
|
|
2352
2709
|
} catch {
|
|
2353
2710
|
continue;
|
|
2354
2711
|
}
|
|
@@ -2451,6 +2808,7 @@ export {
|
|
|
2451
2808
|
queryAccountBreakdown,
|
|
2452
2809
|
openDatabase,
|
|
2453
2810
|
normalizeModelName,
|
|
2811
|
+
mergePeerDatabase,
|
|
2454
2812
|
listSubscriptions,
|
|
2455
2813
|
listProjects,
|
|
2456
2814
|
listModelPricing,
|