@oar-sh/oar 0.9.18 → 0.9.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/extensions/web-relay/polling/polling-loop.mjs +2 -26
- package/.github/extensions/web-relay/server-lifecycle/managed-server.mjs +13 -1
- package/.github/extensions/web-relay/skills/question-routing-hooks.mjs +2 -26
- package/CHANGELOG.md +359 -0
- package/README.md +10 -10
- package/bin/oar.js +391 -97
- package/package.json +4 -2
- package/server/claude-cloud-worker/claude-cloud-event-normalizer.mjs +60 -0
- package/server/claude-cloud-worker/claude-cloud-session-process.mjs +27 -0
- package/server/claude-worker/claude-remote-relay-tool.mjs +3 -2
- package/server/claude-worker/claude-session-process.mjs +3 -0
- package/server/claude-worker/claude-turn-publisher.mjs +163 -8
- package/server/claude-worker/claude-usage-totals.mjs +58 -0
- package/server/copilot-worker/copilot-plan-board.mjs +4 -7
- package/server/copilot-worker/copilot-sdk-event-normalizer.mjs +86 -3
- package/server/copilot-worker/copilot-sdk-session-process.mjs +57 -4
- package/server/cursor-worker/cursor-turn-runner.mjs +42 -6
- package/server/db-schema.mjs +18 -0
- package/server/deepseek-worker/deepseek-claude-backend.mjs +14 -0
- package/server/grok-worker/grok-turn-runner.mjs +44 -6
- package/server/migrations/0001-add-structured-answer.mjs +5 -7
- package/server/migrations/0002-recent-workspace-roots-path-key.mjs +5 -7
- package/server/migrations/0003-push-subscriptions.mjs +5 -7
- package/server/migrations/0004-steer-settle-markers.mjs +5 -7
- package/server/migrations/0005-remote-relays.mjs +5 -7
- package/server/migrations/0006-workspace-root-settings.mjs +5 -7
- package/server/migrations/0007-claude-cloud.mjs +5 -7
- package/server/migrations/0008-message-pins.mjs +5 -7
- package/server/migrations/0009-subagent-timeline.mjs +5 -7
- package/server/migrations/0010-subagent-timeline-pruning.mjs +5 -7
- package/server/migrations/0011-relay-event-subscriptions.mjs +5 -7
- package/server/migrations/0012-cross-relay-event-subscriptions.mjs +5 -7
- package/server/migrations/0013-previews.mjs +5 -7
- package/server/migrations/0014-conversation-groups.mjs +5 -7
- package/server/migrations/0015-conversation-group-position.mjs +5 -7
- package/server/migrations/0016-share-access-log.mjs +5 -7
- package/server/migrations/0017-conversation-history-indexes.mjs +5 -7
- package/server/migrations/0018-background-task-rows.mjs +5 -7
- package/server/migrations/0019-relay-event-subscription-options.mjs +5 -7
- package/server/migrations/0020-conversation-may-create-sessions.mjs +5 -7
- package/server/migrations/0021-conversation-group-agent-marks.mjs +5 -7
- package/server/migrations/0022-relay-schedules.mjs +5 -7
- package/server/migrations/0023-relay-asks.mjs +5 -7
- package/server/migrations/0024-deepseek-usage-totals.mjs +5 -7
- package/server/migrations/0025-deepseek-usage-requests-daily.mjs +5 -7
- package/server/migrations/0026-deepseek-usage-served-models.mjs +5 -7
- package/server/migrations/0027-deepseek-usage-checkpoint-runs.mjs +5 -7
- package/server/migrations/0028-deepseek-usage-checkpoint-request-runs.mjs +5 -7
- package/server/migrations/0029-deepseek-balance-rises.mjs +5 -7
- package/server/migrations/0030-deepseek-usage-actuals.mjs +5 -7
- package/server/migrations/0031-queue-provider.mjs +5 -7
- package/server/migrations/0032-runtime-session-billing-switch.mjs +5 -7
- package/server/migrations/0033-claude-api-usage-checkpoints.mjs +5 -7
- package/server/migrations/0034-claude-api-usage-totals.mjs +5 -7
- package/server/migrations/0035-conversation-plan-only.mjs +87 -0
- package/server/migrations/0036-token-usage.mjs +267 -0
- package/server/public/app/about-view.mjs +87 -3
- package/server/public/app/api-client.js +2 -1
- package/server/public/app/bootstrap.js +55 -2
- package/server/public/app/composer-control-state.mjs +12 -2
- package/server/public/app/context-usage-view.mjs +16 -0
- package/server/public/app/conversation-view.js +17 -2
- package/server/public/app/inbox-ui.js +32 -0
- package/server/public/app/inbox-view.mjs +141 -5
- package/server/public/app/plan-board-shared.mjs +49 -0
- package/server/public/app/plan-usage-view.mjs +1 -1
- package/server/public/app/relay-board-inline.mjs +4 -1
- package/server/public/app/relay-board-view.js +38 -10
- package/server/public/app/relay-event-view.mjs +4 -0
- package/server/public/app/socket-handlers.js +10 -2
- package/server/public/index.html +14 -0
- package/server/public/sw.js +8 -0
- package/server/relay-heap-guard.mjs +351 -0
- package/server/relay-self-restart.mjs +116 -9
- package/server/relay-tools.md +5 -4
- package/server/repositories/message-repository.mjs +31 -1
- package/server/repositories/remote-relay-repository.mjs +10 -0
- package/server/repositories/session-repository.mjs +6 -1
- package/server/routes/messages-routes.mjs +120 -4
- package/server/routes/relay-ask-routes.mjs +138 -8
- package/server/routes/relay-board-routes.mjs +55 -40
- package/server/routes/sessions-routes.mjs +141 -31
- package/server/server-runtime.mjs +142 -11
- package/server/services/about-service.mjs +59 -0
- package/server/services/cache-rebuild-service.mjs +7 -0
- package/server/services/conversation-preferences-service.mjs +31 -0
- package/server/services/database-file.mjs +413 -0
- package/server/services/db-backup-service.mjs +809 -0
- package/server/services/deepseek-usage-import.mjs +6 -9
- package/server/services/delete-archive-service.mjs +3 -0
- package/server/services/earlier-database-service.mjs +69 -126
- package/server/services/oar-cli-helpers.mjs +214 -12
- package/server/services/oar-cli-menu.mjs +1 -0
- package/server/services/oar-state-migration-service.mjs +33 -13
- package/server/services/package-data-rescue.mjs +4 -2
- package/server/services/plan-usage-claude-api.mjs +14 -15
- package/server/services/plan-usage-contract.mjs +6 -2
- package/server/services/plan-usage-deepseek.mjs +1 -1
- package/server/services/plan-usage-tokens.mjs +248 -0
- package/server/services/push-dispatch-service.mjs +134 -9
- package/server/services/relay-cli-launcher-service.mjs +3 -0
- package/server/services/relay-event-service.mjs +30 -0
- package/server/services/relay-question-context.mjs +11 -0
- package/server/services/relay-restart-picture.mjs +67 -22
- package/server/services/remote-inbox-watch-service.mjs +204 -0
- package/server/services/remote-relay-caller-context.mjs +80 -7
- package/server/services/remote-relay-dispatcher.mjs +68 -8
- package/server/services/restart-resume-service.mjs +5 -2
- package/server/services/session-worker-launch-service.mjs +12 -3
- package/server/services/session-worker-stop-service.mjs +27 -6
- package/server/services/token-usage-service.mjs +430 -0
- package/server/services/update-install-service.mjs +13 -0
- package/server/services/usage-limit-pause-service.mjs +8 -1
- package/server/services/worker-scope.mjs +77 -0
- package/shared/plan-board-actions.mjs +130 -0
- package/shared/plan-only.mjs +43 -0
- package/shared/relay-asks.mjs +3 -0
- package/shared/relay-events.mjs +38 -4
- package/shared/remote-relay-contract.mjs +98 -21
- package/shared/turn-token-usage.mjs +144 -0
|
@@ -26,7 +26,7 @@
|
|
|
26
26
|
* DeepSeek (`readActuals`), so the other keys of the account stay out of it.
|
|
27
27
|
* The relay's own usage tables are never touched.
|
|
28
28
|
*
|
|
29
|
-
* CLI: npm run deepseek:import -- amount.csv cost.csv [path/to/
|
|
29
|
+
* CLI: npm run deepseek:import -- amount.csv cost.csv [path/to/oar.db]
|
|
30
30
|
*/
|
|
31
31
|
|
|
32
32
|
import { fileURLToPath } from 'url';
|
|
@@ -35,6 +35,7 @@ import path from 'path';
|
|
|
35
35
|
import process from 'process';
|
|
36
36
|
import Database from 'better-sqlite3';
|
|
37
37
|
|
|
38
|
+
import { cliDbPath } from './database-file.mjs';
|
|
38
39
|
import { DEEPSEEK_API_KEY_SETTING_KEY } from './deepseek-settings-service.mjs';
|
|
39
40
|
import { DEEPSEEK_USAGE_ACTUALS_TABLE, migrateDeepSeekUsageActuals } from '../migrations/0030-deepseek-usage-actuals.mjs';
|
|
40
41
|
|
|
@@ -415,13 +416,8 @@ export function createDeepSeekUsageImportService({ db, getApiKey = () => '', now
|
|
|
415
416
|
return { importExport, readActuals };
|
|
416
417
|
}
|
|
417
418
|
|
|
418
|
-
function defaultDbPath() {
|
|
419
|
-
const here = path.dirname(fileURLToPath(import.meta.url));
|
|
420
|
-
return path.resolve(here, '..', 'data', 'copilot.db');
|
|
421
|
-
}
|
|
422
|
-
|
|
423
419
|
/** The CLI's work: reads the files, imports them into the database at `dbPath`. */
|
|
424
|
-
export function importDeepSeekUsageFiles({ amountPath, costPath, dbPath =
|
|
420
|
+
export function importDeepSeekUsageFiles({ amountPath, costPath, dbPath = cliDbPath(null, new URL('..', import.meta.url)), now = () => new Date() } = {}) {
|
|
425
421
|
const amount = fs.readFileSync(amountPath, 'utf8');
|
|
426
422
|
const cost = fs.readFileSync(costPath, 'utf8');
|
|
427
423
|
const db = new Database(dbPath);
|
|
@@ -444,11 +440,12 @@ const isMain = process.argv[1] && path.resolve(process.argv[1]) === fileURLToPat
|
|
|
444
440
|
if (isMain) {
|
|
445
441
|
const [amountArg, costArg, dbArg] = process.argv.slice(2);
|
|
446
442
|
if (!amountArg || !costArg) {
|
|
447
|
-
console.error('Usage: npm run deepseek:import -- amount.csv cost.csv [path/to/
|
|
443
|
+
console.error('Usage: npm run deepseek:import -- amount.csv cost.csv [path/to/oar.db]');
|
|
448
444
|
process.exit(2);
|
|
449
445
|
}
|
|
450
|
-
const dbPath = dbArg ? path.resolve(dbArg) : defaultDbPath();
|
|
451
446
|
try {
|
|
447
|
+
// The given path, else the database in server/data; never a new one.
|
|
448
|
+
const dbPath = cliDbPath(dbArg, new URL('..', import.meta.url));
|
|
452
449
|
const result = importDeepSeekUsageFiles({ amountPath: path.resolve(amountArg), costPath: path.resolve(costArg), dbPath });
|
|
453
450
|
if (!result.ok) {
|
|
454
451
|
console.error(`DeepSeek usage import refused: ${result.error}`);
|
|
@@ -24,6 +24,7 @@ function createSdkDeleteSessionCaller(sdkClient) {
|
|
|
24
24
|
}
|
|
25
25
|
|
|
26
26
|
import { cleanupGeneratedImagesForConversation } from './generated-image-cleanup-service.mjs';
|
|
27
|
+
import { forgetTurnTokenRows } from './token-usage-service.mjs';
|
|
27
28
|
|
|
28
29
|
export function createDeleteArchiveService(db, sdkClient, { resolveSessionStateRoot = null, conversationGroups = null } = {}) {
|
|
29
30
|
const callDeleteSession = createSdkDeleteSessionCaller(sdkClient);
|
|
@@ -55,6 +56,8 @@ export function createDeleteArchiveService(db, sdkClient, { resolveSessionStateR
|
|
|
55
56
|
db.prepare(`DELETE FROM queue WHERE conversation_id = ?`).run(conversationId);
|
|
56
57
|
db.prepare(`DELETE FROM messages WHERE conversation_id = ?`).run(conversationId);
|
|
57
58
|
db.prepare(`DELETE FROM runtime_sessions WHERE conversation_id = ?`).run(conversationId);
|
|
59
|
+
// Its turns' tokens go too; the day sums stay.
|
|
60
|
+
forgetTurnTokenRows(db, [conversationId]);
|
|
58
61
|
db.prepare(`DELETE FROM conversations WHERE id = ?`).run(conversationId);
|
|
59
62
|
}),
|
|
60
63
|
};
|
|
@@ -4,12 +4,13 @@ import os from 'os';
|
|
|
4
4
|
import path from 'path';
|
|
5
5
|
|
|
6
6
|
import { resolveDataDir, resolveUploadDir } from './data-dir-resolution.mjs';
|
|
7
|
+
import {
|
|
8
|
+
DB_FILE_NAMES, LEGACY_DB_FILE_NAME, newerOf, oneLine, readDatabase, renameWithRetry, resolveDatabaseFile, setAsideDatabase, sideFiles, sleepSync,
|
|
9
|
+
} from './database-file.mjs';
|
|
7
10
|
import { resolveOarRoot } from './oar-state-migration-service.mjs';
|
|
8
11
|
import { resolveStateLayout } from './oar-state-layout.mjs';
|
|
9
12
|
import { defaultIsProcessAlive, lockHolderMayRun, processStartedAt as defaultProcessStartedAt } from './relay-singleton-guard.mjs';
|
|
10
13
|
|
|
11
|
-
const DB_FILE = 'copilot.db';
|
|
12
|
-
const DB_FILES = [DB_FILE, `${DB_FILE}-wal`, `${DB_FILE}-shm`];
|
|
13
14
|
const SHA256_NAME = /^[a-f0-9]{64}$/;
|
|
14
15
|
// The install-time rescue's copies (package-data-rescue.mjs), in the data folder.
|
|
15
16
|
export const RESCUE_DIR_NAME = 'rescued-package-data';
|
|
@@ -26,43 +27,9 @@ const STAGING_PREFIX = '.earlier-database-';
|
|
|
26
27
|
// kill cut short is never taken for one; and where it came from.
|
|
27
28
|
const STAGED_CHECKED_FILE = '.checked';
|
|
28
29
|
const STAGED_SOURCE_FILE = 'source.json';
|
|
29
|
-
// The latest change a database records, per table: what tells which of two
|
|
30
|
-
// databases was used last.
|
|
31
|
-
const STAMPS = [
|
|
32
|
-
['conversations', 'updated_at'],
|
|
33
|
-
['messages', 'timestamp'],
|
|
34
|
-
['app_settings', 'updated_at'],
|
|
35
|
-
];
|
|
36
|
-
// Defender or the indexer may hold a file for a moment at boot.
|
|
37
|
-
const RENAME_RETRY_MS = [50, 100, 200, 400, 800];
|
|
38
|
-
const RETRY_CODES = new Set(['EPERM', 'EBUSY', 'EACCES']);
|
|
39
30
|
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
}
|
|
43
|
-
|
|
44
|
-
function renameWithRetry(fsImpl, from, to, sleep = sleepSync) {
|
|
45
|
-
for (let attempt = 0; ; attempt += 1) {
|
|
46
|
-
try {
|
|
47
|
-
fsImpl.renameSync(from, to);
|
|
48
|
-
return;
|
|
49
|
-
} catch (error) {
|
|
50
|
-
if (!RETRY_CODES.has(error?.code) || attempt >= RENAME_RETRY_MS.length) throw error;
|
|
51
|
-
sleep(RENAME_RETRY_MS[attempt]);
|
|
52
|
-
}
|
|
53
|
-
}
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
// A quick_check report is many lines: the start and nothing more.
|
|
57
|
-
function oneLine(text) {
|
|
58
|
-
return String(text || '').split(/\r?\n/).map((line) => line.trim()).filter(Boolean).slice(0, 2).join(' ').slice(0, 200);
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
function parseStamp(value) {
|
|
62
|
-
if (value === null || value === undefined || value === '') return null;
|
|
63
|
-
const ms = Date.parse(String(value));
|
|
64
|
-
return Number.isFinite(ms) ? ms : null;
|
|
65
|
-
}
|
|
31
|
+
// Read and compared here and when both database names are in one folder.
|
|
32
|
+
export { newerOf, readDatabase };
|
|
66
33
|
|
|
67
34
|
function hashToken(value) {
|
|
68
35
|
const token = String(value || '').trim();
|
|
@@ -87,53 +54,6 @@ function writeJson(fsImpl, file, value) {
|
|
|
87
54
|
renameWithRetry(fsImpl, temporary, file);
|
|
88
55
|
}
|
|
89
56
|
|
|
90
|
-
function readDatabase(Database, dbPath, { readonly = false, singleFile = false, tick = () => {} } = {}) {
|
|
91
|
-
const db = new Database(dbPath, readonly ? { readonly: true, fileMustExist: true } : { fileMustExist: true });
|
|
92
|
-
try {
|
|
93
|
-
const integrity = String(db.pragma('quick_check', { simple: true }) || '');
|
|
94
|
-
const has = (table) => !!db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?").get(table);
|
|
95
|
-
const stamps = {};
|
|
96
|
-
const latest = {};
|
|
97
|
-
for (const [table, column] of STAMPS) {
|
|
98
|
-
// Parsed per row, not compared as text: a value that is no time tells nothing.
|
|
99
|
-
let newest = null;
|
|
100
|
-
let newestText = null;
|
|
101
|
-
if (has(table)) {
|
|
102
|
-
let rows = 0;
|
|
103
|
-
for (const row of db.prepare(`SELECT ${column} AS value FROM ${table}`).iterate()) {
|
|
104
|
-
if (++rows % 10000 === 0) tick();
|
|
105
|
-
const ms = parseStamp(row.value);
|
|
106
|
-
if (ms !== null && (newest === null || ms > newest)) { newest = ms; newestText = String(row.value); }
|
|
107
|
-
}
|
|
108
|
-
}
|
|
109
|
-
stamps[table] = newest;
|
|
110
|
-
latest[table] = newestText;
|
|
111
|
-
}
|
|
112
|
-
const count = (table) => (has(table) ? Number(db.prepare(`SELECT COUNT(*) AS n FROM ${table}`).get()?.n || 0) : 0);
|
|
113
|
-
// The copy goes into place as one file: no -wal or -shm to move with it.
|
|
114
|
-
if (singleFile && integrity === 'ok') db.pragma('journal_mode = DELETE');
|
|
115
|
-
return { integrity, stamps, latest, conversations: count('conversations'), messages: count('messages') };
|
|
116
|
-
} finally {
|
|
117
|
-
db.close();
|
|
118
|
-
}
|
|
119
|
-
}
|
|
120
|
-
|
|
121
|
-
// Which database has the later changes: 'earlier', 'pinned', 'same', or
|
|
122
|
-
// 'both' when each has a table the other changed less recently.
|
|
123
|
-
function newerOf(earlier, pinned) {
|
|
124
|
-
let earlierNewer = 0;
|
|
125
|
-
let pinnedNewer = 0;
|
|
126
|
-
for (const [table] of STAMPS) {
|
|
127
|
-
const a = earlier[table];
|
|
128
|
-
const b = pinned ? pinned[table] : null;
|
|
129
|
-
if (a !== null && (b === null || a > b)) earlierNewer += 1;
|
|
130
|
-
if (b !== null && (a === null || b > a)) pinnedNewer += 1;
|
|
131
|
-
}
|
|
132
|
-
if (earlierNewer && pinnedNewer) return 'both';
|
|
133
|
-
if (earlierNewer) return 'earlier';
|
|
134
|
-
return pinnedNewer ? 'pinned' : 'same';
|
|
135
|
-
}
|
|
136
|
-
|
|
137
57
|
/**
|
|
138
58
|
* Before the database opens, under the relay's lock: the databases a relay
|
|
139
59
|
* pinned to the state root's data folder may have run on until now (the one
|
|
@@ -192,7 +112,6 @@ function adopt({
|
|
|
192
112
|
if (!layout.dataDir || normalize(dataDir) !== normalize(layout.dataDir)) {
|
|
193
113
|
return { status: 'not-applicable', reason: 'own-folders' };
|
|
194
114
|
}
|
|
195
|
-
const pinnedDb = path.join(dataDir, DB_FILE);
|
|
196
115
|
const stamp = now().toISOString().replace(/[:.]/g, '-');
|
|
197
116
|
const rename = (from, to) => renameWithRetry(fsImpl, from, to, sleep);
|
|
198
117
|
|
|
@@ -203,8 +122,14 @@ function adopt({
|
|
|
203
122
|
|
|
204
123
|
const checkedFile = path.join(dataDir, CHECKED_FILE);
|
|
205
124
|
const checked = readJson(fsImpl, checkedFile)?.sources || [];
|
|
125
|
+
// A folder's database under either name (oar.db first), or null.
|
|
126
|
+
// A link to nothing is no database: skipped, as before 0.9.20.
|
|
127
|
+
const dbNameIn = (dir) => {
|
|
128
|
+
const file = resolveDatabaseFile(dir, { fsImpl });
|
|
129
|
+
return file.created || file.dangling.length ? null : file.name;
|
|
130
|
+
};
|
|
206
131
|
const signatureOf = (dbPath) => {
|
|
207
|
-
const files =
|
|
132
|
+
const files = sideFiles(path.basename(dbPath)).map((name) => path.join(path.dirname(dbPath), name));
|
|
208
133
|
let size = 0;
|
|
209
134
|
let mtimeMs = 0;
|
|
210
135
|
for (const file of files) {
|
|
@@ -233,10 +158,13 @@ function adopt({
|
|
|
233
158
|
// A copy the rescue did not finish is not a rescue.
|
|
234
159
|
if (name.endsWith('.partial')) continue;
|
|
235
160
|
const dbDir = path.join(rescueRoot, name, 'data');
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
161
|
+
const dbName = dbNameIn(dbDir);
|
|
162
|
+
// One without a database is recorded under the name old packages had.
|
|
163
|
+
const sourceDb = path.join(dbDir, dbName || LEGACY_DB_FILE_NAME);
|
|
164
|
+
if (seen(signatureOf(sourceDb))) continue;
|
|
165
|
+
const rescue = { kind: 'rescue', dbDir, dbName, sourceDb, uploadDirs: [path.join(rescueRoot, name, 'uploads')], cursorDir: path.join(dbDir, 'cursor-agents') };
|
|
166
|
+
if (dbName) candidates.push(rescue);
|
|
167
|
+
else if (!readdir(fsImpl, dbDir).some((file) => DB_FILE_NAMES.some((each) => file.startsWith(each)))) filesOnly.push(rescue);
|
|
240
168
|
}
|
|
241
169
|
// 2. Copies an interrupted run left, checked complete: possibly the only
|
|
242
170
|
// one there is. They take the upload and Cursor folders of their source.
|
|
@@ -244,12 +172,15 @@ function adopt({
|
|
|
244
172
|
try { leftovers = fsImpl.readdirSync(dataDir).filter((name) => name.startsWith(STAGING_PREFIX)).sort(); } catch {}
|
|
245
173
|
for (const name of leftovers) {
|
|
246
174
|
const dbDir = path.join(dataDir, name);
|
|
247
|
-
|
|
248
|
-
if (
|
|
175
|
+
const dbName = dbNameIn(dbDir);
|
|
176
|
+
if (!exists(path.join(dbDir, STAGED_CHECKED_FILE)) || !dbName) continue;
|
|
177
|
+
if (seen(signatureOf(path.join(dbDir, dbName)))) continue;
|
|
249
178
|
const source = readJson(fsImpl, path.join(dbDir, STAGED_SOURCE_FILE)) || {};
|
|
250
179
|
candidates.push({
|
|
251
180
|
kind: 'leftover',
|
|
252
181
|
dbDir,
|
|
182
|
+
dbName,
|
|
183
|
+
sourceDb: path.join(dbDir, dbName),
|
|
253
184
|
uploadDirs: Array.isArray(source.uploadDirs) ? source.uploadDirs : [],
|
|
254
185
|
cursorDir: source.cursorDir || null,
|
|
255
186
|
});
|
|
@@ -257,8 +188,9 @@ function adopt({
|
|
|
257
188
|
// 3. What the relay ran on without the pin (`server/data` or a config
|
|
258
189
|
// `dataDir`, copied from, never renamed); never the pinned folder itself.
|
|
259
190
|
const earlierDir = resolveDataDir({ env: unpinned, configPath, serverDir });
|
|
260
|
-
const
|
|
261
|
-
|
|
191
|
+
const earlierName = normalize(earlierDir) !== normalize(dataDir) ? dbNameIn(earlierDir) : null;
|
|
192
|
+
const earlierDb = earlierName ? path.join(earlierDir, earlierName) : null;
|
|
193
|
+
if (earlierDb && !seen(signatureOf(earlierDb))) {
|
|
262
194
|
const lock = readJson(fsImpl, path.join(earlierDir, 'relay-server.lock'));
|
|
263
195
|
const defaultRoot = resolveOarRoot({ ...env, OAR_STATE_ROOT: '' }, platform);
|
|
264
196
|
const ownRoot = normalize(layout.root) === normalize(defaultRoot)
|
|
@@ -269,14 +201,14 @@ function adopt({
|
|
|
269
201
|
path.join(serverDir, 'uploads'),
|
|
270
202
|
resolveUploadDir({ dataDir: earlierDir, env: unpinned, serverDir, platform }),
|
|
271
203
|
].filter((dir, index, all) => all.findIndex((other) => normalize(other) === normalize(dir)) === index);
|
|
272
|
-
candidates.push({ kind: 'earlier', dbDir: earlierDir, uploadDirs, cursorDir: path.join(earlierDir, 'cursor-agents'), lock });
|
|
204
|
+
candidates.push({ kind: 'earlier', dbDir: earlierDir, dbName: earlierName, sourceDb: earlierDb, uploadDirs, cursorDir: path.join(earlierDir, 'cursor-agents'), lock });
|
|
273
205
|
}
|
|
274
206
|
}
|
|
275
207
|
for (const candidate of filesOnly) {
|
|
276
208
|
const filled = fillMissing({ candidate, dataDir, uploadDir, fsImpl, exists, normalize });
|
|
277
209
|
// Looked at once it all came over; a failed copy is tried again.
|
|
278
210
|
if (!filled.ok) continue;
|
|
279
|
-
checked.push({ ...signatureOf(
|
|
211
|
+
checked.push({ ...signatureOf(candidate.sourceDb), outcome: 'files', at: now().toISOString() });
|
|
280
212
|
try { writeJson(fsImpl, checkedFile, { sources: checked }); } catch {}
|
|
281
213
|
logger.log?.(`[data] Copied ${filled.uploads} upload(s) and ${filled.cursorFiles} Cursor file(s) of ${path.dirname(candidate.dbDir)} where missing.`);
|
|
282
214
|
}
|
|
@@ -284,7 +216,9 @@ function adopt({
|
|
|
284
216
|
|
|
285
217
|
const results = [];
|
|
286
218
|
for (const [index, candidate] of candidates.entries()) {
|
|
287
|
-
const sourceDb =
|
|
219
|
+
const { sourceDb } = candidate;
|
|
220
|
+
// Looked up per candidate: one adopted before may have placed it.
|
|
221
|
+
const pinned = resolveDatabaseFile(dataDir, { fsImpl });
|
|
288
222
|
const progress = trackProgress({
|
|
289
223
|
sourceDb, bytes: signatureOf(sourceDb).size, pid, clock, logger,
|
|
290
224
|
write: (value) => writeJson(fsImpl, path.join(dataDir, TAKEOVER_PROGRESS_FILE), value),
|
|
@@ -295,7 +229,7 @@ function adopt({
|
|
|
295
229
|
backupDir: path.join(layout.root, 'backups', candidates.length > 1 ? `${stamp}-${index}` : stamp),
|
|
296
230
|
dataDir,
|
|
297
231
|
uploadDir,
|
|
298
|
-
|
|
232
|
+
pinnedName: pinned.created ? null : pinned.name,
|
|
299
233
|
Database,
|
|
300
234
|
fsImpl,
|
|
301
235
|
rename,
|
|
@@ -317,23 +251,23 @@ function adopt({
|
|
|
317
251
|
const packageRoot = normalize(path.dirname(serverDir));
|
|
318
252
|
const ours = candidate.kind !== 'earlier' || normalize(candidate.dbDir).startsWith(packageRoot + path.sep);
|
|
319
253
|
let renamedTo = null;
|
|
320
|
-
if (ours)
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
}
|
|
325
|
-
renamedTo =
|
|
326
|
-
}
|
|
254
|
+
if (ours) {
|
|
255
|
+
// Under its own name, `<name>.older-<time>` or `<name>.adopted-<time>`:
|
|
256
|
+
// checkpointed, the -wal/-shm first and the main file last; a failure
|
|
257
|
+
// leaves it where it was (it is looked at again only if it changes).
|
|
258
|
+
const aside = setAsideDatabase({ dir: candidate.dbDir, name: candidate.dbName, suffix, Database, rename, fsImpl });
|
|
259
|
+
if (aside.ok) renamedTo = aside.renamedTo;
|
|
260
|
+
}
|
|
327
261
|
checked.push({ ...signature, outcome: result.status, at: now().toISOString(), latest: result.latest, renamedTo });
|
|
328
262
|
try { writeJson(fsImpl, checkedFile, { sources: checked }); } catch {}
|
|
329
263
|
if (result.status === 'adopted') {
|
|
330
264
|
logger.log?.(
|
|
331
|
-
`[data] Brought over the database this relay ran on until now, ${sourceDb} (${result.conversations} conversations, ${result.messages} messages), to ${
|
|
265
|
+
`[data] Brought over the database this relay ran on until now, ${sourceDb} (${result.conversations} conversations, ${result.messages} messages), to ${result.dbPath}`
|
|
332
266
|
+ `${result.uploads ? ` with ${result.uploads} upload(s)` : ''}; the original is kept${renamedTo ? ` as ${renamedTo}` : ` at ${sourceDb}`}`
|
|
333
267
|
+ `${result.backupDir ? `, the older database that was there in ${result.backupDir}` : ''}.`,
|
|
334
268
|
);
|
|
335
269
|
} else {
|
|
336
|
-
logger.log?.(`[data] ${sourceDb} is not newer than ${
|
|
270
|
+
logger.log?.(`[data] ${sourceDb} is not newer than ${result.dbPath}; left it${renamedTo ? ` as ${renamedTo}` : ''}, not used.`);
|
|
337
271
|
}
|
|
338
272
|
results.push({ ...result, from: candidate.dbDir, renamedTo, cursorFiles: filled.cursorFiles });
|
|
339
273
|
}
|
|
@@ -427,19 +361,21 @@ function finishInterruptedSwap({ dataDir, fsImpl, rename, exists, logger }) {
|
|
|
427
361
|
const journalPath = path.join(dataDir, SWAP_JOURNAL_FILE);
|
|
428
362
|
const journal = readJson(fsImpl, journalPath);
|
|
429
363
|
if (!journal) return null;
|
|
430
|
-
|
|
364
|
+
// A journal 0.9.19 wrote has no name: its database had the name before 0.9.20.
|
|
365
|
+
const dbName = journal.dbName || LEGACY_DB_FILE_NAME;
|
|
366
|
+
const staged = path.join(String(journal.stagingDir || ''), dbName);
|
|
431
367
|
try {
|
|
432
368
|
if (exists(staged)) {
|
|
433
369
|
// The copy is not in place yet: what is still in the data folder is
|
|
434
370
|
// the pinned database, and goes where the journal put the rest of it.
|
|
435
371
|
fsImpl.mkdirSync(journal.backupDir, { recursive: true });
|
|
436
|
-
for (const name of
|
|
372
|
+
for (const name of sideFiles(dbName)) {
|
|
437
373
|
if (exists(path.join(dataDir, name))) rename(path.join(dataDir, name), path.join(journal.backupDir, name));
|
|
438
374
|
}
|
|
439
|
-
rename(staged, path.join(dataDir,
|
|
440
|
-
} else if (!exists(path.join(dataDir,
|
|
375
|
+
rename(staged, path.join(dataDir, dbName));
|
|
376
|
+
} else if (!exists(path.join(dataDir, dbName)) && exists(path.join(journal.backupDir, dbName))) {
|
|
441
377
|
// Neither the copy nor a database in place: the pinned one goes back.
|
|
442
|
-
for (const name of
|
|
378
|
+
for (const name of sideFiles(dbName)) {
|
|
443
379
|
if (exists(path.join(journal.backupDir, name))) rename(path.join(journal.backupDir, name), path.join(dataDir, name));
|
|
444
380
|
}
|
|
445
381
|
}
|
|
@@ -457,9 +393,13 @@ function finishInterruptedSwap({ dataDir, fsImpl, rename, exists, logger }) {
|
|
|
457
393
|
}
|
|
458
394
|
|
|
459
395
|
function considerCandidate({
|
|
460
|
-
candidate, stagingDir, backupDir, dataDir, uploadDir,
|
|
396
|
+
candidate, stagingDir, backupDir, dataDir, uploadDir, pinnedName, Database, fsImpl, rename, exists, lockMayRun, normalize, progress,
|
|
461
397
|
}) {
|
|
462
|
-
const sourceDb =
|
|
398
|
+
const { sourceDb } = candidate;
|
|
399
|
+
// It takes the pinned database's name; with none there it keeps its own,
|
|
400
|
+
// so a database under the name before 0.9.20 stays one an earlier version runs on.
|
|
401
|
+
const dbName = pinnedName || candidate.dbName;
|
|
402
|
+
const pinnedDb = path.join(dataDir, dbName);
|
|
463
403
|
const refused = (reason) => ({
|
|
464
404
|
status: 'refused',
|
|
465
405
|
from: candidate.dbDir,
|
|
@@ -473,12 +413,14 @@ function considerCandidate({
|
|
|
473
413
|
try {
|
|
474
414
|
progress.step('copy');
|
|
475
415
|
fsImpl.mkdirSync(stagingDir, { recursive: true });
|
|
476
|
-
|
|
477
|
-
|
|
416
|
+
// Staged under the name it goes in as, -wal and -shm with it.
|
|
417
|
+
const stagedNames = sideFiles(dbName);
|
|
418
|
+
for (const [i, name] of sideFiles(candidate.dbName).entries()) {
|
|
419
|
+
if (exists(path.join(candidate.dbDir, name))) fsImpl.copyFileSync(path.join(candidate.dbDir, name), path.join(stagingDir, stagedNames[i]));
|
|
478
420
|
}
|
|
479
421
|
// The copy, never the original: opening it applies its -wal.
|
|
480
422
|
progress.step('check');
|
|
481
|
-
earlier = readDatabase(Database, path.join(stagingDir,
|
|
423
|
+
earlier = readDatabase(Database, path.join(stagingDir, dbName), { singleFile: true, tick: progress.tick });
|
|
482
424
|
} catch (error) {
|
|
483
425
|
removeStaging();
|
|
484
426
|
return refused(`its database could not be copied and read (${error?.message || error}).`);
|
|
@@ -495,17 +437,17 @@ function considerCandidate({
|
|
|
495
437
|
return refused(`its database could not be copied and read (${error?.message || error}).`);
|
|
496
438
|
}
|
|
497
439
|
let pinned = null;
|
|
498
|
-
if (
|
|
440
|
+
if (pinnedName) {
|
|
499
441
|
// Read through a copy as well: opening it, even read-only, would rebuild
|
|
500
442
|
// its -shm, and a refused start must leave it as it was.
|
|
501
443
|
const pinnedCopy = `${stagingDir}-pinned`;
|
|
502
444
|
try {
|
|
503
445
|
progress.step('pinned');
|
|
504
446
|
fsImpl.mkdirSync(pinnedCopy, { recursive: true });
|
|
505
|
-
for (const name of
|
|
447
|
+
for (const name of sideFiles(dbName)) {
|
|
506
448
|
if (exists(path.join(dataDir, name))) fsImpl.copyFileSync(path.join(dataDir, name), path.join(pinnedCopy, name));
|
|
507
449
|
}
|
|
508
|
-
pinned = readDatabase(Database, path.join(pinnedCopy,
|
|
450
|
+
pinned = readDatabase(Database, path.join(pinnedCopy, dbName), { tick: progress.tick });
|
|
509
451
|
} catch (error) {
|
|
510
452
|
removeStaging();
|
|
511
453
|
return refused(`the pinned database could not be read (${error?.message || error}).`);
|
|
@@ -520,11 +462,11 @@ function considerCandidate({
|
|
|
520
462
|
const newer = newerOf(earlier.stamps, pinned?.stamps);
|
|
521
463
|
if (newer === 'both') {
|
|
522
464
|
removeStaging();
|
|
523
|
-
return refused(
|
|
465
|
+
return refused(`each database has changes newer than the other's. Move the files of the one not to go on with (${sideFiles(candidate.dbName).join(', ')} in ${candidate.dbDir}, or ${sideFiles(dbName).join(', ')} in ${dataDir}) into a folder of your own, then start again.`);
|
|
524
466
|
}
|
|
525
467
|
if (newer !== 'earlier') {
|
|
526
468
|
removeStaging();
|
|
527
|
-
return { status: 'kept', latest: earlier.latest };
|
|
469
|
+
return { status: 'kept', dbPath: pinnedDb, latest: earlier.latest };
|
|
528
470
|
}
|
|
529
471
|
|
|
530
472
|
// Uploads first: the database about to come over refers to them.
|
|
@@ -550,14 +492,14 @@ function considerCandidate({
|
|
|
550
492
|
const movedAside = [];
|
|
551
493
|
try {
|
|
552
494
|
progress.step('swap');
|
|
553
|
-
writeJson(fsImpl, journalPath, { stagingDir, backupDir, at: new Date().toISOString() });
|
|
554
|
-
for (const name of
|
|
495
|
+
writeJson(fsImpl, journalPath, { stagingDir, backupDir, dbName, at: new Date().toISOString() });
|
|
496
|
+
for (const name of sideFiles(dbName)) {
|
|
555
497
|
if (!exists(path.join(dataDir, name))) continue;
|
|
556
498
|
fsImpl.mkdirSync(backupDir, { recursive: true });
|
|
557
499
|
rename(path.join(dataDir, name), path.join(backupDir, name));
|
|
558
500
|
movedAside.push(name);
|
|
559
501
|
}
|
|
560
|
-
rename(path.join(stagingDir,
|
|
502
|
+
rename(path.join(stagingDir, dbName), pinnedDb);
|
|
561
503
|
} catch (error) {
|
|
562
504
|
// Back as it was. Only the pinned files this run moved go back; the copy
|
|
563
505
|
// is still in staging, so nothing of it is in the data folder.
|
|
@@ -580,6 +522,7 @@ function considerCandidate({
|
|
|
580
522
|
|
|
581
523
|
return {
|
|
582
524
|
status: 'adopted',
|
|
525
|
+
dbPath: pinnedDb,
|
|
583
526
|
backupDir: movedAside.length ? backupDir : null,
|
|
584
527
|
uploads: addedUploads.length,
|
|
585
528
|
conversations: earlier.conversations,
|