@wirecat/cli-core 0.594.0 → 0.596.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -1
- package/dist/background/schedules.d.ts +10 -1
- package/dist/background/schedules.js +42 -5
- package/dist/cli/index.d.ts +1 -1
- package/dist/cli/messenger/store-maintenance-command.js +54 -5
- package/dist/cli/schedule/schedule-command.d.ts +4 -2
- package/dist/cli/schedule/schedule-command.js +3 -1
- package/dist/entities/index.d.ts +14 -2
- package/dist/entities/store-maintenance.d.ts +14 -2
- package/dist/entities/store-maintenance.js +19 -6
- package/dist/store/migrations.d.ts +11 -2
- package/dist/store/migrations.js +32 -2
- package/dist/store/sqlite/manifest.d.ts +2 -0
- package/dist/store/sqlite/manifest.js +57 -5
- package/dist/store/sqlite/open-store.d.ts +2 -2
- package/dist/store/sqlite/open-store.js +37 -7
- package/dist/store/sqlite/read-compatibility.d.ts +2 -0
- package/dist/store/sqlite/read-compatibility.js +7 -3
- package/dist/store/sqlite/rollback-copy.d.ts +49 -0
- package/dist/store/sqlite/rollback-copy.js +290 -0
- package/dist/store/store.d.ts +1 -0
- package/dist/store/store.js +1 -0
- package/package.json +7 -1
package/CHANGELOG.md
CHANGED
|
@@ -5,7 +5,34 @@ Notable changes to `@wirecat/cli-core` (`@wirecat/cli-messaging` up to 0.261.0,
|
|
|
5
5
|
minor version may break callers, and says how under "Changed — may break callers". `pnpm docs:check` checks the
|
|
6
6
|
shape of this file.
|
|
7
7
|
|
|
8
|
-
##
|
|
8
|
+
## 0.596.0 — 06.11.2026
|
|
9
|
+
|
|
10
|
+
### Added
|
|
11
|
+
|
|
12
|
+
- **`package.json` says which store schema the version speaks.** `wirecat.store` holds `speaks`, the newest schema
|
|
13
|
+
it migrates a file to, and `minCompatible`, the oldest schema that can still use a file it migrated, so an
|
|
14
|
+
installer can check a tool against the owner's store from the npm registry before installing it.
|
|
15
|
+
|
|
16
|
+
### Fixed
|
|
17
|
+
|
|
18
|
+
- **A read-only open migrates an older store first.** `openStore({ readOnly: true })` on a file at an older schema
|
|
19
|
+
opens it the ordinary way once, which migrates it, then opens it read-only, instead of refusing it with
|
|
20
|
+
`configuration_error`. A tool whose first command only reads no longer needs a workaround. A newer incompatible
|
|
21
|
+
schema is still refused, and a missing file is still not created.
|
|
22
|
+
|
|
23
|
+
## 0.595.0 — 06.11.2026
|
|
24
|
+
|
|
25
|
+
### Added
|
|
26
|
+
|
|
27
|
+
- **A rollback copy before a breaking store upgrade.** Before a migration that raises `min_compatible` (after which
|
|
28
|
+
older tools refuse the store), the base tables it names are copied into `backups/` beside the store, as
|
|
29
|
+
`<store>-rollback-schema-<version>-<time>.db`, with the schema as it was. Search indexes, vectors and other derived
|
|
30
|
+
tables are never copied. Without the free space for it, the upgrade runs anyway and warns. `store restore <copy>`
|
|
31
|
+
puts the tables back and returns the store to that version, so the older tool opens it again; its `store rebuild`
|
|
32
|
+
then refreshes the search indexes. The copy is deleted on the first open a day after the upgrade succeeded.
|
|
33
|
+
- **`Migration.tables`** and the manifest's `tables`: required on a migration that raises `minCompatible`, the base
|
|
34
|
+
tables it changes by their names before it. `generatedMigrations` refuses a raising one without it, or one that
|
|
35
|
+
names a derived table.
|
|
9
36
|
|
|
10
37
|
## 0.594.0 — 06.11.2026
|
|
11
38
|
|
|
@@ -49,6 +49,14 @@ export interface DaySpend {
|
|
|
49
49
|
export declare const localDay: (at: Date) => string;
|
|
50
50
|
/** What the schedule's runs cost today; nothing before its first priced run of the day. */
|
|
51
51
|
export declare const spentToday: (file: string, now?: Date) => number;
|
|
52
|
+
/**
|
|
53
|
+
* What an agent is given on its stdin before its first turn: the host's own reads of what the run is about, so
|
|
54
|
+
* the agent spends no model turns fetching them. Returns `undefined` when the schedule has none. `run` runs one
|
|
55
|
+
* command of this CLI under the run's variables and answers its stdout.
|
|
56
|
+
*/
|
|
57
|
+
export type AgentInput = (schedule: Schedule, run: (argv: string[]) => Promise<string>) => Promise<string | undefined>;
|
|
58
|
+
/** The most an agent's input may hold; a longer one is cut, so a mailbox in flood cannot grow every turn's context. */
|
|
59
|
+
export declare const AGENT_INPUT_MAX_CHARS = 60000;
|
|
52
60
|
/** Another CLI a scheduled agent may run by name, as the host declares it: `memo`'s agent runs `mail`. */
|
|
53
61
|
export type AgentCli = Pick<AppIdentity, "command" | "appName" | "envPrefix">;
|
|
54
62
|
export interface LastRun {
|
|
@@ -164,7 +172,7 @@ export declare const CREDENTIALS_UNREACHABLE = "credentials.unreachable";
|
|
|
164
172
|
* keyring, so a run that fails for that reason is tried again, a minute then up to 15 minutes apart, for up to
|
|
165
173
|
* 12 hours. A schedule that repeats every few minutes is left to its next tick; a run by hand fails at once.
|
|
166
174
|
*/
|
|
167
|
-
export declare const runSchedule: (app: AppIdentity, system: ServerSystem, env: NodeJS.ProcessEnv, file: string, { now, timer, signal, killAfterMs, untilUnlocked, wait, agentRuns, }?: {
|
|
175
|
+
export declare const runSchedule: (app: AppIdentity, system: ServerSystem, env: NodeJS.ProcessEnv, file: string, { now, timer, signal, killAfterMs, untilUnlocked, wait, agentRuns, agentInput, }?: {
|
|
168
176
|
now?: () => Date;
|
|
169
177
|
timer?: Timer;
|
|
170
178
|
signal?: AbortSignal;
|
|
@@ -173,4 +181,5 @@ export declare const runSchedule: (app: AppIdentity, system: ServerSystem, env:
|
|
|
173
181
|
wait?: (ms: number, signal?: AbortSignal) => Promise<void>;
|
|
174
182
|
/** The other CLIs an agent may run; each runs under its own read-only lock and knows it runs from a schedule. */
|
|
175
183
|
agentRuns?: readonly AgentCli[];
|
|
184
|
+
agentInput?: AgentInput;
|
|
176
185
|
}) => Promise<LastRun>;
|
|
@@ -53,6 +53,11 @@ const reportOf = (stdout) => {
|
|
|
53
53
|
return { text: stdout };
|
|
54
54
|
}
|
|
55
55
|
};
|
|
56
|
+
/** The most an agent's input may hold; a longer one is cut, so a mailbox in flood cannot grow every turn's context. */
|
|
57
|
+
export const AGENT_INPUT_MAX_CHARS = 60_000;
|
|
58
|
+
const capped = (input) => input.length <= AGENT_INPUT_MAX_CHARS
|
|
59
|
+
? input
|
|
60
|
+
: `${input.slice(0, AGENT_INPUT_MAX_CHARS)}\n\n[input cut at ${AGENT_INPUT_MAX_CHARS} characters; the rest waits for the next run]\n`;
|
|
56
61
|
/** Why a timer left a run out, as an id a translation can key on; `why` stays the English sentence. */
|
|
57
62
|
export const SKIP_REASONS = {
|
|
58
63
|
costDayCap: "cost.day-cap",
|
|
@@ -426,7 +431,7 @@ const pause = (ms, signal) => new Promise((resolve) => {
|
|
|
426
431
|
* keyring, so a run that fails for that reason is tried again, a minute then up to 15 minutes apart, for up to
|
|
427
432
|
* 12 hours. A schedule that repeats every few minutes is left to its next tick; a run by hand fails at once.
|
|
428
433
|
*/
|
|
429
|
-
export const runSchedule = async (app, system, env, file, { now = () => new Date(), timer, signal, killAfterMs = KILL_AFTER_MS, untilUnlocked = false, wait = pause, agentRuns = [], } = {}) => {
|
|
434
|
+
export const runSchedule = async (app, system, env, file, { now = () => new Date(), timer, signal, killAfterMs = KILL_AFTER_MS, untilUnlocked = false, wait = pause, agentRuns = [], agentInput, } = {}) => {
|
|
430
435
|
const schedule = readSchedule(file);
|
|
431
436
|
const folder = join(file, "..");
|
|
432
437
|
const runs = join(folder, "runs");
|
|
@@ -434,12 +439,24 @@ export const runSchedule = async (app, system, env, file, { now = () => new Date
|
|
|
434
439
|
const agent = schedule.kind === "agent";
|
|
435
440
|
const retries = untilUnlocked && !("every" in schedule.when);
|
|
436
441
|
const firstAt = now().getTime();
|
|
437
|
-
let last = await attemptSchedule(app, system, env, schedule, folder, {
|
|
442
|
+
let last = await attemptSchedule(app, system, env, schedule, folder, {
|
|
443
|
+
now,
|
|
444
|
+
signal,
|
|
445
|
+
killAfterMs,
|
|
446
|
+
agentRuns,
|
|
447
|
+
agentInput,
|
|
448
|
+
});
|
|
438
449
|
for (let delay = LOCKED_FIRST_WAIT_MS; retries && lockedOut(last) && !signal?.aborted && now().getTime() - firstAt + delay <= LOCKED_GIVE_UP_MS; delay = Math.min(delay * 2, LOCKED_LONGEST_WAIT_MS)) {
|
|
439
450
|
await wait(delay, signal);
|
|
440
451
|
if (signal?.aborted)
|
|
441
452
|
break;
|
|
442
|
-
last = await attemptSchedule(app, system, env, schedule, folder, {
|
|
453
|
+
last = await attemptSchedule(app, system, env, schedule, folder, {
|
|
454
|
+
now,
|
|
455
|
+
signal,
|
|
456
|
+
killAfterMs,
|
|
457
|
+
agentRuns,
|
|
458
|
+
agentInput,
|
|
459
|
+
});
|
|
443
460
|
}
|
|
444
461
|
const { exitCode, output } = last;
|
|
445
462
|
const errors = agent ? output.replace(/\.md$/, ".log") : output;
|
|
@@ -493,8 +510,24 @@ export const runSchedule = async (app, system, env, file, { now = () => new Date
|
|
|
493
510
|
}
|
|
494
511
|
return last;
|
|
495
512
|
};
|
|
513
|
+
/** The agent's input, or none: a host read that fails leaves the agent to fetch what it needs itself. */
|
|
514
|
+
const inputOf = async (agentInput, system, env, schedule, log) => {
|
|
515
|
+
try {
|
|
516
|
+
const input = await agentInput(schedule, async (argv) => {
|
|
517
|
+
const ran = await system.run([...system.entry, ...argv], env);
|
|
518
|
+
if (ran.code !== 0)
|
|
519
|
+
throw new Error(`${argv.slice(0, 3).join(" ")} exited ${ran.code}`);
|
|
520
|
+
return ran.stdout;
|
|
521
|
+
});
|
|
522
|
+
return input === undefined ? undefined : capped(input);
|
|
523
|
+
}
|
|
524
|
+
catch (error) {
|
|
525
|
+
log.warn({ event: "schedule.input-failed", schedule: schedule.name, reason: String(error) });
|
|
526
|
+
return undefined;
|
|
527
|
+
}
|
|
528
|
+
};
|
|
496
529
|
/** One run of the schedule's command or agent: its output in `runs/`, its result in `last.json`. */
|
|
497
|
-
const attemptSchedule = async (app, system, env, schedule, folder, { now, signal, killAfterMs, agentRuns, }) => {
|
|
530
|
+
const attemptSchedule = async (app, system, env, schedule, folder, { now, signal, killAfterMs, agentRuns, agentInput, }) => {
|
|
498
531
|
const runs = join(folder, "runs");
|
|
499
532
|
const stamp = now()
|
|
500
533
|
.toISOString()
|
|
@@ -526,9 +559,13 @@ const attemptSchedule = async (app, system, env, schedule, folder, { now, signal
|
|
|
526
559
|
const facts = { schedule: schedule.name, kind: schedule.kind };
|
|
527
560
|
const began = now().getTime();
|
|
528
561
|
log.info({ event: "schedule.start", ...facts });
|
|
562
|
+
const input = agent && agentInput ? await inputOf(agentInput, system, childEnv, schedule, log) : undefined;
|
|
529
563
|
let costUsd;
|
|
530
564
|
const exitCode = await new Promise((resolve) => {
|
|
531
|
-
const child = spawn(program, args, { env: childEnv, cwd: folder, stdio: ["
|
|
565
|
+
const child = spawn(program, args, { env: childEnv, cwd: folder, stdio: ["pipe", "pipe", "pipe"] });
|
|
566
|
+
// An agent that stops reading its stdin may close it first; that is no failure of the run.
|
|
567
|
+
child.stdin?.on("error", () => { });
|
|
568
|
+
child.stdin?.end(input);
|
|
532
569
|
const out = [];
|
|
533
570
|
const err = [];
|
|
534
571
|
child.stdout.on("data", (chunk) => out.push(chunk));
|
package/dist/cli/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
export { AI_ENTRIES, AI_SETTING_KEYS, type AISettings, resolveAISettings } from "../analysis/settings.js";
|
|
2
|
-
export type { AgentCli } from "../background/schedules.js";
|
|
2
|
+
export type { AgentCli, AgentInput } from "../background/schedules.js";
|
|
3
3
|
export { parseDuration } from "../domain/duration.js";
|
|
4
4
|
export { openOf, PROFILE_PLACEHOLDER, type ProfilesFor, type RefOpen, type RefSummary, refLabel, } from "../domain/ref-summaries.js";
|
|
5
5
|
export { botTemplate } from "../entities/list-results.js";
|
|
@@ -12,7 +12,7 @@ import { migrate } from "../../store/migrations.js";
|
|
|
12
12
|
import { openCache } from "../../store/open.js";
|
|
13
13
|
import { storePath } from "../../store/path.js";
|
|
14
14
|
import { deleteCopy, repairStore } from "../../store/repair.js";
|
|
15
|
-
import { drainNoteIndex, dropAllUnusedVectors, fillSearchIndex, fillStems, involvementStoreOver, noteIndexState, openSqlite, openStore, orphanPointers, pendingNormalization, resetAttachmentWords, resetNoteIndex, resetSearchIndex, resetStems, searchIndexState, stemmerCache, stemsState, vectorIndexCounts, } from "../../store/store.js";
|
|
15
|
+
import { drainNoteIndex, dropAllUnusedVectors, fillSearchIndex, fillStems, involvementStoreOver, noteIndexState, openSqlite, openStore, orphanPointers, pendingNormalization, resetAttachmentWords, resetNoteIndex, resetSearchIndex, resetStems, rollBack, searchIndexState, stemmerCache, stemsState, vectorIndexCounts, } from "../../store/store.js";
|
|
16
16
|
import { environmentOf, outputFor } from "../context.js";
|
|
17
17
|
import { destroysNothing, LOCAL_READ, LOCAL_WRITE } from "../effects.js";
|
|
18
18
|
import { itemOf, renderAnswer, snakeKeys } from "../envelope.js";
|
|
@@ -362,7 +362,11 @@ const migrateCommand = (messenger) => annotate(new Command("migrate"), { output:
|
|
|
362
362
|
}
|
|
363
363
|
const answer = await reading(path, (database) => {
|
|
364
364
|
const from = schemaOf(database).version;
|
|
365
|
-
migrate(database, {
|
|
365
|
+
migrate(database, {
|
|
366
|
+
command: messenger.app.command,
|
|
367
|
+
version: messenger.app.version,
|
|
368
|
+
rollback: { store: path, warn: (line) => renderer.warn(line) },
|
|
369
|
+
});
|
|
366
370
|
const { normalized, indexed, terms } = buildWordIndex(database, (note) => renderer.note(note));
|
|
367
371
|
const stemmed = buildStems(database, (note) => renderer.note(note));
|
|
368
372
|
const notes = buildNoteIndex(database, (note) => renderer.note(note));
|
|
@@ -533,9 +537,11 @@ const decryptCommand = () => annotate(new Command("decrypt"), { output: { entity
|
|
|
533
537
|
* `serve` runs: that opens the store on its first write, so until then it holds nothing to see.
|
|
534
538
|
*/
|
|
535
539
|
const restoreCommand = (messenger) => annotate(new Command("restore"), { output: { entity: "store_restore" }, effects: LOCAL_WRITE })
|
|
536
|
-
.summary("restore a backup
|
|
537
|
-
.description("put a backup in place of the store; the store it replaces is kept beside it, never deleted"
|
|
538
|
-
|
|
540
|
+
.summary("restore a backup and keep the current store, or roll back a breaking upgrade")
|
|
541
|
+
.description("put a backup in place of the store; the store it replaces is kept beside it, never deleted. Given a rollback " +
|
|
542
|
+
"copy (backups/<store>-rollback-schema-<version>-<time>.db, kept for a day before an upgrade older tools " +
|
|
543
|
+
"cannot open), put its tables back in the store and return it to that version instead")
|
|
544
|
+
.argument("<file>", "a file `store backup` wrote, or a rollback copy; one written with --encrypt asks for its password")
|
|
539
545
|
.action(async function (file) {
|
|
540
546
|
const path = storePath(environmentOf(this).env ?? process.env);
|
|
541
547
|
const given = resolve(file);
|
|
@@ -560,6 +566,8 @@ const restoreFrom = async (command, messenger, backup, given) => {
|
|
|
560
566
|
if (existsSync(path) && realpathSync(backup) === realpathSync(path)) {
|
|
561
567
|
throw new CliError("validation_error", `${backup} is the store itself`);
|
|
562
568
|
}
|
|
569
|
+
if (await isRollbackCopy(backup))
|
|
570
|
+
return rollBackFrom(command, messenger, backup);
|
|
563
571
|
const schema = await backupSchema(backup);
|
|
564
572
|
const stamp = stampNow();
|
|
565
573
|
const kept = existsSync(path) ? `${path}.before-restore-${stamp}` : null;
|
|
@@ -588,6 +596,47 @@ const restoreFrom = async (command, messenger, backup, given) => {
|
|
|
588
596
|
renderer.note(`the backup is behind this build; the next command migrates it — \`${lineOf(command, "store migrate")}\` now`);
|
|
589
597
|
}
|
|
590
598
|
};
|
|
599
|
+
const isRollbackCopy = async (file) => {
|
|
600
|
+
try {
|
|
601
|
+
return await readingOnly(file, (database) => database.prepare("SELECT 1 FROM sqlite_master WHERE name = 'rollback_meta'").get() !== undefined);
|
|
602
|
+
}
|
|
603
|
+
catch {
|
|
604
|
+
return false;
|
|
605
|
+
}
|
|
606
|
+
};
|
|
607
|
+
/** In place, in one transaction: the copy holds only the tables the upgrade changed, never a whole store. */
|
|
608
|
+
const rollBackFrom = async (command, messenger, copy) => {
|
|
609
|
+
const { renderer } = outputFor(command);
|
|
610
|
+
const path = storePath(environmentOf(command).env ?? process.env);
|
|
611
|
+
if (!existsSync(path))
|
|
612
|
+
throw new CliError("not_found", `no store at ${path} to roll back`);
|
|
613
|
+
const waitMs = environmentOf(command).storeLockWaitMs;
|
|
614
|
+
await quiesce(path, messenger, "restore", (message) => renderer.warn(message), waitMs);
|
|
615
|
+
const { database } = await openSqlite(path);
|
|
616
|
+
let rolled;
|
|
617
|
+
try {
|
|
618
|
+
rolled = rollBack(database, copy);
|
|
619
|
+
}
|
|
620
|
+
finally {
|
|
621
|
+
database.close();
|
|
622
|
+
}
|
|
623
|
+
result(command, {
|
|
624
|
+
path,
|
|
625
|
+
restoredFrom: copy,
|
|
626
|
+
rolledBack: true,
|
|
627
|
+
schema: rolled.from,
|
|
628
|
+
was: rolled.was,
|
|
629
|
+
tables: rolled.restored,
|
|
630
|
+
rebuilt: rolled.rebuilt,
|
|
631
|
+
foreignKeyViolations: rolled.foreignKeyViolations,
|
|
632
|
+
});
|
|
633
|
+
const taken = new Date(rolled.createdAt).toISOString();
|
|
634
|
+
renderer.note(`the store is back at schema ${rolled.from}; ${rolled.tables.length > 0 ? rolled.tables.join(", ") : "no table"} ` +
|
|
635
|
+
`hold what they held at ${taken}, and what was written to them since is not kept`);
|
|
636
|
+
renderer.note("open it with the older tool and run its `store rebuild`: the search indexes are behind the tables");
|
|
637
|
+
renderer.note(`a tool that speaks schema ${rolled.to} or later migrates it again on its next run`);
|
|
638
|
+
renderer.note("restart every serve and mcp of either CLI that was running, so they read the restored store");
|
|
639
|
+
};
|
|
591
640
|
const backupSchema = async (backup) => {
|
|
592
641
|
let read;
|
|
593
642
|
try {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { Command } from "commander";
|
|
2
|
-
import { type AgentCli, type Schedule } from "../../background/schedules.js";
|
|
2
|
+
import { type AgentCli, type AgentInput, type Schedule } from "../../background/schedules.js";
|
|
3
3
|
import { type MailWaiting } from "../../mail/marks.js";
|
|
4
4
|
import { type AppIdentity } from "../../ports/app.js";
|
|
5
5
|
import { type AnswerOutput } from "../envelope.js";
|
|
@@ -19,6 +19,8 @@ export interface ScheduleHost {
|
|
|
19
19
|
* then runs only once enough of it waits, or the oldest has waited long enough.
|
|
20
20
|
*/
|
|
21
21
|
newMail?: (command: Command, holder: string) => Promise<MailWaiting>;
|
|
22
|
+
/** What an agent is given on its stdin before its first turn; see `AgentInput`. */
|
|
23
|
+
agentInput?: AgentInput;
|
|
22
24
|
/** The config file's settings for agent schedules; a schedule's own flags win, the defaults fill the rest. */
|
|
23
25
|
limits?: (command: Command) => ScheduleLimits;
|
|
24
26
|
/**
|
|
@@ -47,5 +49,5 @@ export declare const scheduleLimitsOf: (config: {
|
|
|
47
49
|
* (macOS) or Task Scheduler (Windows). Every timer runs `schedule exec <file>`; the schedule file holds
|
|
48
50
|
* the rest, so what runs and with which variables is the same on all three.
|
|
49
51
|
*/
|
|
50
|
-
export declare const scheduleCommandFor: ({ app, context: contextOf, newMail, limits: configured, agentRuns, }: ScheduleHost) => Command;
|
|
52
|
+
export declare const scheduleCommandFor: ({ app, context: contextOf, newMail, agentInput, limits: configured, agentRuns, }: ScheduleHost) => Command;
|
|
51
53
|
export {};
|
|
@@ -157,7 +157,7 @@ const runCapOf = (argv) => {
|
|
|
157
157
|
* (macOS) or Task Scheduler (Windows). Every timer runs `schedule exec <file>`; the schedule file holds
|
|
158
158
|
* the rest, so what runs and with which variables is the same on all three.
|
|
159
159
|
*/
|
|
160
|
-
export const scheduleCommandFor = ({ app, context: contextOf, newMail, limits: configured, agentRuns = [], }) => {
|
|
160
|
+
export const scheduleCommandFor = ({ app, context: contextOf, newMail, agentInput, limits: configured, agentRuns = [], }) => {
|
|
161
161
|
const agentClis = [app, ...agentRuns];
|
|
162
162
|
const command = new Command("schedule").description(`run a ${app.command} command or an agent later or on a timer (systemd, launchd, Task Scheduler); an agent cannot send`);
|
|
163
163
|
const prepare = (self, resolving = {}) => {
|
|
@@ -427,6 +427,7 @@ export const scheduleCommandFor = ({ app, context: contextOf, newMail, limits: c
|
|
|
427
427
|
const last = await untilChildEnds(this, (signal) => runSchedule(app, system, context.env, scheduleFile(state, scheduleName(name)), {
|
|
428
428
|
timer: timer(),
|
|
429
429
|
agentRuns,
|
|
430
|
+
...(agentInput ? { agentInput } : {}),
|
|
430
431
|
...(signal ? { signal } : {}),
|
|
431
432
|
}));
|
|
432
433
|
say(snakeKeys({ name, ...last }), [`${name}: exit ${last.exitCode} — ${last.output}`]);
|
|
@@ -462,6 +463,7 @@ export const scheduleCommandFor = ({ app, context: contextOf, newMail, limits: c
|
|
|
462
463
|
const last = await untilChildEnds(this, (signal) => runSchedule(app, system, context.env, file, {
|
|
463
464
|
timer: timer(),
|
|
464
465
|
agentRuns,
|
|
466
|
+
...(agentInput ? { agentInput } : {}),
|
|
465
467
|
untilUnlocked: true,
|
|
466
468
|
...(signal ? { signal } : {}),
|
|
467
469
|
}));
|
package/dist/entities/index.d.ts
CHANGED
|
@@ -9649,12 +9649,24 @@ export declare const ENTITIES: {
|
|
|
9649
9649
|
readonly from: v.StringSchema<undefined>;
|
|
9650
9650
|
readonly bytes: v.SchemaWithPipe<readonly [v.NumberSchema<undefined>, v.IntegerAction<number, undefined>, v.MinValueAction<number, 0, undefined>]>;
|
|
9651
9651
|
}, undefined>;
|
|
9652
|
-
readonly store_restore: v.StrictObjectSchema<{
|
|
9652
|
+
readonly store_restore: v.UnionSchema<[v.StrictObjectSchema<{
|
|
9653
9653
|
readonly path: v.StringSchema<undefined>;
|
|
9654
9654
|
readonly restored_from: v.StringSchema<undefined>;
|
|
9655
9655
|
readonly kept_at: v.NullableSchema<v.StringSchema<undefined>, undefined>;
|
|
9656
9656
|
readonly schema: v.SchemaWithPipe<readonly [v.NumberSchema<undefined>, v.IntegerAction<number, undefined>, v.MinValueAction<number, 0, undefined>]>;
|
|
9657
|
-
}, undefined
|
|
9657
|
+
}, undefined>, v.StrictObjectSchema<{
|
|
9658
|
+
readonly path: v.StringSchema<undefined>;
|
|
9659
|
+
readonly restored_from: v.StringSchema<undefined>;
|
|
9660
|
+
readonly rolled_back: v.LiteralSchema<true, undefined>;
|
|
9661
|
+
readonly schema: v.SchemaWithPipe<readonly [v.NumberSchema<undefined>, v.IntegerAction<number, undefined>, v.MinValueAction<number, 0, undefined>]>;
|
|
9662
|
+
readonly was: v.SchemaWithPipe<readonly [v.NumberSchema<undefined>, v.IntegerAction<number, undefined>, v.MinValueAction<number, 0, undefined>]>;
|
|
9663
|
+
readonly tables: v.ArraySchema<v.StrictObjectSchema<{
|
|
9664
|
+
readonly table: v.StringSchema<undefined>;
|
|
9665
|
+
readonly rows: v.SchemaWithPipe<readonly [v.NumberSchema<undefined>, v.IntegerAction<number, undefined>, v.MinValueAction<number, 0, undefined>]>;
|
|
9666
|
+
}, undefined>, undefined>;
|
|
9667
|
+
readonly rebuilt: v.ArraySchema<v.StringSchema<undefined>, undefined>;
|
|
9668
|
+
readonly foreign_key_violations: v.SchemaWithPipe<readonly [v.NumberSchema<undefined>, v.IntegerAction<number, undefined>, v.MinValueAction<number, 0, undefined>]>;
|
|
9669
|
+
}, undefined>], undefined>;
|
|
9658
9670
|
readonly store_reset: v.UnionSchema<[v.StrictObjectSchema<{
|
|
9659
9671
|
readonly path: v.StringSchema<undefined>;
|
|
9660
9672
|
readonly exists: v.LiteralSchema<false, undefined>;
|
|
@@ -344,12 +344,24 @@ export declare const storeDecrypt: v.StrictObjectSchema<{
|
|
|
344
344
|
readonly from: v.StringSchema<undefined>;
|
|
345
345
|
readonly bytes: v.SchemaWithPipe<readonly [v.NumberSchema<undefined>, v.IntegerAction<number, undefined>, v.MinValueAction<number, 0, undefined>]>;
|
|
346
346
|
}, undefined>;
|
|
347
|
-
export declare const storeRestore: v.StrictObjectSchema<{
|
|
347
|
+
export declare const storeRestore: v.UnionSchema<[v.StrictObjectSchema<{
|
|
348
348
|
readonly path: v.StringSchema<undefined>;
|
|
349
349
|
readonly restored_from: v.StringSchema<undefined>;
|
|
350
350
|
readonly kept_at: v.NullableSchema<v.StringSchema<undefined>, undefined>;
|
|
351
351
|
readonly schema: v.SchemaWithPipe<readonly [v.NumberSchema<undefined>, v.IntegerAction<number, undefined>, v.MinValueAction<number, 0, undefined>]>;
|
|
352
|
-
}, undefined
|
|
352
|
+
}, undefined>, v.StrictObjectSchema<{
|
|
353
|
+
readonly path: v.StringSchema<undefined>;
|
|
354
|
+
readonly restored_from: v.StringSchema<undefined>;
|
|
355
|
+
readonly rolled_back: v.LiteralSchema<true, undefined>;
|
|
356
|
+
readonly schema: v.SchemaWithPipe<readonly [v.NumberSchema<undefined>, v.IntegerAction<number, undefined>, v.MinValueAction<number, 0, undefined>]>;
|
|
357
|
+
readonly was: v.SchemaWithPipe<readonly [v.NumberSchema<undefined>, v.IntegerAction<number, undefined>, v.MinValueAction<number, 0, undefined>]>;
|
|
358
|
+
readonly tables: v.ArraySchema<v.StrictObjectSchema<{
|
|
359
|
+
readonly table: v.StringSchema<undefined>;
|
|
360
|
+
readonly rows: v.SchemaWithPipe<readonly [v.NumberSchema<undefined>, v.IntegerAction<number, undefined>, v.MinValueAction<number, 0, undefined>]>;
|
|
361
|
+
}, undefined>, undefined>;
|
|
362
|
+
readonly rebuilt: v.ArraySchema<v.StringSchema<undefined>, undefined>;
|
|
363
|
+
readonly foreign_key_violations: v.SchemaWithPipe<readonly [v.NumberSchema<undefined>, v.IntegerAction<number, undefined>, v.MinValueAction<number, 0, undefined>]>;
|
|
364
|
+
}, undefined>], undefined>;
|
|
353
365
|
export declare const storeReset: v.UnionSchema<[v.StrictObjectSchema<{
|
|
354
366
|
readonly path: v.StringSchema<undefined>;
|
|
355
367
|
readonly exists: v.LiteralSchema<false, undefined>;
|
|
@@ -155,12 +155,25 @@ export const storeBackup = v.strictObject({
|
|
|
155
155
|
encrypted: v.optional(v.literal(true)),
|
|
156
156
|
});
|
|
157
157
|
export const storeDecrypt = v.strictObject({ path: text, from: text, bytes: count });
|
|
158
|
-
export const storeRestore = v.
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
158
|
+
export const storeRestore = v.union([
|
|
159
|
+
v.strictObject({
|
|
160
|
+
path: text,
|
|
161
|
+
restored_from: text,
|
|
162
|
+
kept_at: v.nullable(text),
|
|
163
|
+
schema: count,
|
|
164
|
+
}),
|
|
165
|
+
/** From a rollback copy: its tables put back in place, the store returned to `schema` from `was`. */
|
|
166
|
+
v.strictObject({
|
|
167
|
+
path: text,
|
|
168
|
+
restored_from: text,
|
|
169
|
+
rolled_back: v.literal(true),
|
|
170
|
+
schema: count,
|
|
171
|
+
was: count,
|
|
172
|
+
tables: v.array(v.strictObject({ table: text, rows: count })),
|
|
173
|
+
rebuilt: v.array(text),
|
|
174
|
+
foreign_key_violations: count,
|
|
175
|
+
}),
|
|
176
|
+
]);
|
|
164
177
|
export const storeReset = v.union([
|
|
165
178
|
v.strictObject({ ...absent.entries, reset: v.literal(false) }),
|
|
166
179
|
v.strictObject({
|
|
@@ -1,9 +1,15 @@
|
|
|
1
1
|
import type { CacheDatabase } from "./driver.js";
|
|
2
|
+
import { type RollbackCopyOptions } from "./sqlite/rollback-copy.js";
|
|
2
3
|
export interface Migration {
|
|
3
4
|
version: number;
|
|
4
5
|
/** The oldest schema version whose statements still work on a file at this version. */
|
|
5
6
|
minCompatible: number;
|
|
6
7
|
statements: string[];
|
|
8
|
+
/**
|
|
9
|
+
* The base tables it changes, by their names before it. Required when it raises `minCompatible`: they are copied
|
|
10
|
+
* aside before it runs, so `store restore` can take the store back to the version older tools still open.
|
|
11
|
+
*/
|
|
12
|
+
tables?: string[];
|
|
7
13
|
/** Runs before the statements, inside the same transaction. */
|
|
8
14
|
before?: (database: CacheDatabase, now: () => number) => void;
|
|
9
15
|
}
|
|
@@ -12,7 +18,8 @@ export interface Migration {
|
|
|
12
18
|
* schema. **A migration that has reached anyone's file is never edited.** After it: forward-only and additive —
|
|
13
19
|
* no `DROP` and no rebuild of a base table, a new column is nullable or has a default, and every `INSERT` in the
|
|
14
20
|
* store names its columns — so an older `tg` keeps writing to a file a newer `max` has migrated. Only a breaking
|
|
15
|
-
* change raises `minCompatible`, and that is a major version of this package
|
|
21
|
+
* change raises `minCompatible`, and that is a major version of this package; it names the base tables it changes
|
|
22
|
+
* (`tables`), which are copied aside before it runs (`sqlite/rollback-copy.ts`). The search indexes are derived and
|
|
16
23
|
* may be dropped and rebuilt. Every version after the first gets a `version-N.test.ts` that opens a file at N-1.
|
|
17
24
|
*
|
|
18
25
|
* Times are epoch milliseconds: an integer sorts and ranges without a format to agree on.
|
|
@@ -24,11 +31,13 @@ export declare const MIGRATIONS: Migration[];
|
|
|
24
31
|
* that fails names `store reset` when the app has one: without it the owner is stuck until a build that can migrate
|
|
25
32
|
* the file. An app without that command (drive) passes no `command`, and the advice names none.
|
|
26
33
|
*/
|
|
27
|
-
export declare const migrate: (database: CacheDatabase, { migrations, now, command, version, ownTable, }?: {
|
|
34
|
+
export declare const migrate: (database: CacheDatabase, { migrations, now, command, version, ownTable, rollback, }?: {
|
|
28
35
|
migrations?: Migration[];
|
|
29
36
|
now?: () => number;
|
|
30
37
|
command?: string;
|
|
31
38
|
/** The CLI's own version: with `command`, it is recorded so a later upgrade can name who it locks out. */
|
|
32
39
|
version?: string;
|
|
33
40
|
ownTable?: string | null;
|
|
41
|
+
/** A store on disk: before a migration that raises `minCompatible`, a copy of the tables it changes. */
|
|
42
|
+
rollback?: RollbackCopyOptions;
|
|
34
43
|
}) => string | undefined;
|
package/dist/store/migrations.js
CHANGED
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
import { CliError } from "@wirecat/cli-base";
|
|
2
2
|
import { lockedOutNotice, recordClient, refusalAdvice, storeClients } from "./sqlite/clients.js";
|
|
3
3
|
import { generatedMigrations } from "./sqlite/manifest.js";
|
|
4
|
+
import { discardRollbackCopy, recordRollbackCopy, takeRollbackCopy, } from "./sqlite/rollback-copy.js";
|
|
4
5
|
/**
|
|
5
6
|
* Every migration is generated by drizzle-kit and numbered in `sqlite/manifest.ts`. Version 1 creates the whole
|
|
6
7
|
* schema. **A migration that has reached anyone's file is never edited.** After it: forward-only and additive —
|
|
7
8
|
* no `DROP` and no rebuild of a base table, a new column is nullable or has a default, and every `INSERT` in the
|
|
8
9
|
* store names its columns — so an older `tg` keeps writing to a file a newer `max` has migrated. Only a breaking
|
|
9
|
-
* change raises `minCompatible`, and that is a major version of this package
|
|
10
|
+
* change raises `minCompatible`, and that is a major version of this package; it names the base tables it changes
|
|
11
|
+
* (`tables`), which are copied aside before it runs (`sqlite/rollback-copy.ts`). The search indexes are derived and
|
|
10
12
|
* may be dropped and rebuilt. Every version after the first gets a `version-N.test.ts` that opens a file at N-1.
|
|
11
13
|
*
|
|
12
14
|
* Times are epoch milliseconds: an integer sorts and ranges without a format to agree on.
|
|
@@ -31,18 +33,44 @@ const fileState = (database) => {
|
|
|
31
33
|
.get();
|
|
32
34
|
return { version: Number(row?.version ?? 0), minCompatible: Number(row?.min_compatible ?? 0) };
|
|
33
35
|
};
|
|
36
|
+
/** The tables every pending migration that raises `minCompatible` names; none for a new file or a foreign one. */
|
|
37
|
+
const copyBeforeBreaking = (database, migrations, { ownTable, now }, options) => {
|
|
38
|
+
if (!hasTable(database, "schema_migrations"))
|
|
39
|
+
return undefined;
|
|
40
|
+
const file = fileState(database);
|
|
41
|
+
if (file.version === 0 || (ownTable !== null && !hasTable(database, ownTable)))
|
|
42
|
+
return undefined;
|
|
43
|
+
let floor = file.minCompatible;
|
|
44
|
+
const tables = [];
|
|
45
|
+
for (const migration of migrations) {
|
|
46
|
+
if (migration.version <= file.version || migration.minCompatible <= floor)
|
|
47
|
+
continue;
|
|
48
|
+
floor = migration.minCompatible;
|
|
49
|
+
for (const table of migration.tables ?? [])
|
|
50
|
+
tables.push(table);
|
|
51
|
+
}
|
|
52
|
+
if (floor === file.minCompatible)
|
|
53
|
+
return undefined;
|
|
54
|
+
return takeRollbackCopy(database, { from: file.version, to: latest(migrations), tables, at: now() }, options);
|
|
55
|
+
};
|
|
34
56
|
/**
|
|
35
57
|
* Brings the file up to this version, or opens a newer one it can still write to, or refuses.
|
|
36
58
|
* Inside `BEGIN IMMEDIATE`, so two processes opening an old file at once migrate it once. A migration
|
|
37
59
|
* that fails names `store reset` when the app has one: without it the owner is stuck until a build that can migrate
|
|
38
60
|
* the file. An app without that command (drive) passes no `command`, and the advice names none.
|
|
39
61
|
*/
|
|
40
|
-
export const migrate = (database, { migrations = MIGRATIONS, now = Date.now, command, version, ownTable = OWN_TABLE, } = {}) => {
|
|
62
|
+
export const migrate = (database, { migrations = MIGRATIONS, now = Date.now, command, version, ownTable = OWN_TABLE, rollback, } = {}) => {
|
|
41
63
|
const ours = latest(migrations);
|
|
64
|
+
let copy = rollback ? copyBeforeBreaking(database, migrations, { ownTable, now }, rollback) : undefined;
|
|
42
65
|
database.exec("BEGIN IMMEDIATE");
|
|
43
66
|
try {
|
|
44
67
|
database.exec(HISTORY);
|
|
45
68
|
const file = fileState(database);
|
|
69
|
+
// Another process migrated the file between the copy and this transaction, and took its own.
|
|
70
|
+
if (copy && copy.from !== file.version) {
|
|
71
|
+
discardRollbackCopy(copy);
|
|
72
|
+
copy = undefined;
|
|
73
|
+
}
|
|
46
74
|
if (file.version > 0 && ownTable !== null && !hasTable(database, ownTable)) {
|
|
47
75
|
throw new CliError("configuration_error", `the message store was written by another, incompatible tool (schema ${file.version} by its own numbering, ` +
|
|
48
76
|
"with no meetings table) — this tool will not open or change it; move it aside to start a new store");
|
|
@@ -73,6 +101,8 @@ export const migrate = (database, { migrations = MIGRATIONS, now = Date.now, com
|
|
|
73
101
|
}
|
|
74
102
|
if (command && version)
|
|
75
103
|
recordClient(database, { command, version, speaks: ours }, now());
|
|
104
|
+
if (copy)
|
|
105
|
+
recordRollbackCopy(database, copy, now());
|
|
76
106
|
database.exec("COMMIT");
|
|
77
107
|
return notice;
|
|
78
108
|
}
|
|
@@ -11,6 +11,8 @@ export interface ManifestEntry {
|
|
|
11
11
|
name: string;
|
|
12
12
|
version: number;
|
|
13
13
|
minCompatible: number;
|
|
14
|
+
/** Required on a version that raises `minCompatible`: the base tables it changes (`Migration.tables`). */
|
|
15
|
+
tables?: string[];
|
|
14
16
|
}
|
|
15
17
|
/** One migration creates the whole schema; every later one is added as the next version. */
|
|
16
18
|
export declare const MANIFEST: ManifestEntry[];
|
|
@@ -1,4 +1,33 @@
|
|
|
1
1
|
import { GENERATED } from "./migrations.generated.js";
|
|
2
|
+
import { isDerivedTable } from "./rollback-copy.js";
|
|
3
|
+
const V31_TABLES = [
|
|
4
|
+
"accounts",
|
|
5
|
+
"account_identities",
|
|
6
|
+
"agent_actions",
|
|
7
|
+
"aliases",
|
|
8
|
+
"attachments",
|
|
9
|
+
"bot_updates",
|
|
10
|
+
"chats",
|
|
11
|
+
"documents",
|
|
12
|
+
"email_threads",
|
|
13
|
+
"emails",
|
|
14
|
+
"facts",
|
|
15
|
+
"links",
|
|
16
|
+
"mail_rules",
|
|
17
|
+
"mailboxes",
|
|
18
|
+
"meeting_series",
|
|
19
|
+
"meetings",
|
|
20
|
+
"memories",
|
|
21
|
+
"messages",
|
|
22
|
+
"notes",
|
|
23
|
+
"projects",
|
|
24
|
+
"proposed_actions",
|
|
25
|
+
"reminders",
|
|
26
|
+
"store_settings",
|
|
27
|
+
"sync_cursors",
|
|
28
|
+
"syncs",
|
|
29
|
+
"taggings",
|
|
30
|
+
];
|
|
2
31
|
/** One migration creates the whole schema; every later one is added as the next version. */
|
|
3
32
|
export const MANIFEST = [
|
|
4
33
|
{ name: "20261010184427_initial", version: 1, minCompatible: 1 },
|
|
@@ -36,14 +65,24 @@ export const MANIFEST = [
|
|
|
36
65
|
{ name: "20261021224625_version-28-chat-state", version: 28, minCompatible: 1 },
|
|
37
66
|
{ name: "20261023105057_version-29-transcript-words", version: 29, minCompatible: 1 },
|
|
38
67
|
{ name: "20261023105101_version-29-transcript-words-triggers", version: 29, minCompatible: 1 },
|
|
39
|
-
{
|
|
68
|
+
{
|
|
69
|
+
name: "20261025064253_version-30-reviewed-tag-consolidation",
|
|
70
|
+
version: 30,
|
|
71
|
+
minCompatible: 30,
|
|
72
|
+
tables: ["tags", "auto_tag_claims"],
|
|
73
|
+
},
|
|
40
74
|
{ name: "20261025064302_version-30-resolved-tag-view", version: 30, minCompatible: 30 },
|
|
41
|
-
{ name: "20261025070402_version-31-drive-account", version: 31, minCompatible: 31 },
|
|
42
|
-
{
|
|
75
|
+
{ name: "20261025070402_version-31-drive-account", version: 31, minCompatible: 31, tables: V31_TABLES },
|
|
76
|
+
{
|
|
77
|
+
name: "20261025071848_version-32-files",
|
|
78
|
+
version: 32,
|
|
79
|
+
minCompatible: 32,
|
|
80
|
+
tables: ["accounts", "documents", "identities", "store_settings"],
|
|
81
|
+
},
|
|
43
82
|
{ name: "20261025071850_version-32-files-data", version: 32, minCompatible: 32 },
|
|
44
83
|
{ name: "20261025072643_version-33-calendars", version: 33, minCompatible: 32 },
|
|
45
84
|
{ name: "20261025072644_version-33-calendars-triggers", version: 33, minCompatible: 32 },
|
|
46
|
-
{ name: "20261025182500_version-34-vector-index", version: 34, minCompatible: 34 },
|
|
85
|
+
{ name: "20261025182500_version-34-vector-index", version: 34, minCompatible: 34, tables: [] },
|
|
47
86
|
{ name: "20261025182501_version-34-vector-index-triggers", version: 34, minCompatible: 34 },
|
|
48
87
|
{ name: "20261027102259_version-35-calendar-neutral", version: 35, minCompatible: 34 },
|
|
49
88
|
{ name: "20261027102302_version-35-calendar-neutral-data", version: 35, minCompatible: 34 },
|
|
@@ -83,12 +122,25 @@ export const generatedMigrations = (manifest = MANIFEST, generated = GENERATED)
|
|
|
83
122
|
const last = migrations.at(-1);
|
|
84
123
|
if (last && last.version !== entry.version && entry.minCompatible < last.minCompatible)
|
|
85
124
|
throw new Error("migration minCompatible cannot decrease");
|
|
125
|
+
const derived = entry.tables?.filter(isDerivedTable) ?? [];
|
|
126
|
+
if (derived.length > 0)
|
|
127
|
+
throw new Error(`migration ${entry.version} names derived tables (${derived.join(", ")}); they are rebuilt, never copied`);
|
|
86
128
|
if (last?.version !== entry.version) {
|
|
87
|
-
|
|
129
|
+
if (last && entry.minCompatible > last.minCompatible && entry.tables === undefined)
|
|
130
|
+
throw new Error(`migration ${entry.version} raises minCompatible, so it names the base tables it changes (tables)`);
|
|
131
|
+
migrations.push({
|
|
132
|
+
version: entry.version,
|
|
133
|
+
minCompatible: entry.minCompatible,
|
|
134
|
+
statements: [...found.statements],
|
|
135
|
+
...(entry.tables ? { tables: [...entry.tables] } : {}),
|
|
136
|
+
});
|
|
88
137
|
}
|
|
89
138
|
else if (last.minCompatible !== entry.minCompatible) {
|
|
90
139
|
throw new Error(`the rows of version ${entry.version} disagree on minCompatible`);
|
|
91
140
|
}
|
|
141
|
+
else if (entry.tables && JSON.stringify(entry.tables) !== JSON.stringify(last.tables)) {
|
|
142
|
+
throw new Error(`the rows of version ${entry.version} disagree on tables`);
|
|
143
|
+
}
|
|
92
144
|
else {
|
|
93
145
|
last.statements.push(...found.statements);
|
|
94
146
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { type DecisionPolicy } from "../decisions.js";
|
|
2
2
|
import { type StoreContext } from "./open.js";
|
|
3
3
|
export interface StoreOptions {
|
|
4
|
-
/** Open an existing compatible store without maintenance; SQLite refuses writes. */
|
|
4
|
+
/** Open an existing compatible store without maintenance, migrating an older one first; SQLite refuses writes. */
|
|
5
5
|
readOnly?: boolean;
|
|
6
6
|
path?: string;
|
|
7
7
|
env?: NodeJS.ProcessEnv;
|
|
@@ -16,4 +16,4 @@ export interface StoreOptions {
|
|
|
16
16
|
decisions?: DecisionPolicy;
|
|
17
17
|
}
|
|
18
18
|
export declare const BACKFILL_ON_OPEN = 5000;
|
|
19
|
-
export declare const openStoreContext: (
|
|
19
|
+
export declare const openStoreContext: (options?: StoreOptions) => Promise<StoreContext>;
|
|
@@ -14,7 +14,8 @@ import { logPruner } from "./log-pruning.js";
|
|
|
14
14
|
import { openSqlite } from "./open.js";
|
|
15
15
|
import { movePaymentFacts } from "./payment-facts.js";
|
|
16
16
|
import { loadProviders } from "./providers.js";
|
|
17
|
-
import { assertReadCompatible } from "./read-compatibility.js";
|
|
17
|
+
import { assertReadCompatible, readCompatibility } from "./read-compatibility.js";
|
|
18
|
+
import { pruneRollbackCopies } from "./rollback-copy.js";
|
|
18
19
|
import { stemsState } from "./stems.js";
|
|
19
20
|
import { fillStems, followDefaultStemmers } from "./stems-fill.js";
|
|
20
21
|
import { moveSuggestedLinks } from "./suggested-links.js";
|
|
@@ -35,7 +36,32 @@ const createWritable = (file) => {
|
|
|
35
36
|
throw new CliError("permission_error", "the message store cannot be written from here; run where it is writable, or set MESSAGING_STORE", { reason: "store.not-writable" });
|
|
36
37
|
}
|
|
37
38
|
};
|
|
38
|
-
|
|
39
|
+
// A new tool's first command may only read; it migrates the file as its first write would.
|
|
40
|
+
const openReadable = async (file, options) => {
|
|
41
|
+
const first = await openSqlite(file, { readOnly: true });
|
|
42
|
+
try {
|
|
43
|
+
if (readCompatibility(first.database, options.command) === "current")
|
|
44
|
+
return first;
|
|
45
|
+
}
|
|
46
|
+
catch (error) {
|
|
47
|
+
first.database.close();
|
|
48
|
+
throw error;
|
|
49
|
+
}
|
|
50
|
+
first.database.close();
|
|
51
|
+
const writable = await openStoreContext({ ...options, path: file, readOnly: false });
|
|
52
|
+
writable.database.close();
|
|
53
|
+
const migrated = await openSqlite(file, { readOnly: true });
|
|
54
|
+
try {
|
|
55
|
+
assertReadCompatible(migrated.database, options.command);
|
|
56
|
+
return migrated;
|
|
57
|
+
}
|
|
58
|
+
catch (error) {
|
|
59
|
+
migrated.database.close();
|
|
60
|
+
throw error;
|
|
61
|
+
}
|
|
62
|
+
};
|
|
63
|
+
export const openStoreContext = async (options = {}) => {
|
|
64
|
+
const { path, env, now = Date.now, command, version, warn = (line) => process.stderr.write(`${line}\n`), readOnly = false, decisions = DEFAULT_DECISION_POLICY, } = options;
|
|
39
65
|
if (typeof readOnly !== "boolean")
|
|
40
66
|
throw new CliError("validation_error", "readOnly must be a boolean");
|
|
41
67
|
await storeCapable();
|
|
@@ -47,15 +73,19 @@ export const openStoreContext = async ({ path, env, now = Date.now, command, ver
|
|
|
47
73
|
}
|
|
48
74
|
else
|
|
49
75
|
createWritable(file);
|
|
50
|
-
const opened = await openSqlite(file, { readOnly });
|
|
76
|
+
const opened = readOnly ? await openReadable(file, options) : await openSqlite(file, { readOnly });
|
|
51
77
|
const database = opened.database;
|
|
52
78
|
try {
|
|
53
|
-
if (readOnly)
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
79
|
+
if (!readOnly) {
|
|
80
|
+
const notice = migrate(database, {
|
|
81
|
+
now,
|
|
82
|
+
command,
|
|
83
|
+
...(version ? { version } : {}),
|
|
84
|
+
rollback: { store: file, warn },
|
|
85
|
+
});
|
|
57
86
|
if (notice)
|
|
58
87
|
warn(notice);
|
|
88
|
+
pruneRollbackCopies(database, now());
|
|
59
89
|
fillFiles(database, now);
|
|
60
90
|
fillFolded(database);
|
|
61
91
|
fillJoinKeys(database);
|
|
@@ -1,2 +1,4 @@
|
|
|
1
1
|
import type { CacheDatabase } from "../driver.js";
|
|
2
|
+
/** "older" when the file needs migrating before a read-only open; throws for every file it may not read. */
|
|
3
|
+
export declare const readCompatibility: (database: CacheDatabase, command?: string) => "current" | "older";
|
|
2
4
|
export declare const assertReadCompatible: (database: CacheDatabase, command?: string) => void;
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { CliError } from "@wirecat/cli-base";
|
|
2
2
|
import { MIGRATIONS } from "../migrations.js";
|
|
3
3
|
import { refusalAdvice, storeClients } from "./clients.js";
|
|
4
|
-
|
|
4
|
+
/** "older" when the file needs migrating before a read-only open; throws for every file it may not read. */
|
|
5
|
+
export const readCompatibility = (database, command) => {
|
|
5
6
|
const ours = MIGRATIONS.at(-1)?.version ?? 0;
|
|
6
7
|
if (!database.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='schema_migrations'").get())
|
|
7
8
|
throw new CliError("configuration_error", "The store has no schema history; initialize it explicitly before a read-only open");
|
|
@@ -15,8 +16,11 @@ export const assertReadCompatible = (database, command) => {
|
|
|
15
16
|
minCompatible < 1 ||
|
|
16
17
|
minCompatible > version)
|
|
17
18
|
throw new CliError("configuration_error", "The store schema history is invalid; read-only opening cannot repair it");
|
|
18
|
-
if (version < ours)
|
|
19
|
-
throw new CliError("configuration_error", `The store needs schema ${ours} maintenance before a read-only open (found ${version})`);
|
|
20
19
|
if (minCompatible > ours)
|
|
21
20
|
throw new CliError("configuration_error", `The store requires schema ${minCompatible}, but this tool supports ${ours}; ${refusalAdvice(storeClients(database), command, version)}`);
|
|
21
|
+
return version < ours ? "older" : "current";
|
|
22
|
+
};
|
|
23
|
+
export const assertReadCompatible = (database, command) => {
|
|
24
|
+
if (readCompatibility(database, command) === "older")
|
|
25
|
+
throw new CliError("configuration_error", `The store needs schema ${MIGRATIONS.at(-1)?.version ?? 0} maintenance before a read-only open`);
|
|
22
26
|
};
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import type { CacheDatabase } from "../driver.js";
|
|
2
|
+
export declare const isDerivedTable: (name: string) => boolean;
|
|
3
|
+
export interface RollbackCopyOptions {
|
|
4
|
+
/** The store's own path: the copy goes in `backups/` beside it. */
|
|
5
|
+
store: string;
|
|
6
|
+
freeBytes?: (dir: string) => number;
|
|
7
|
+
warn?: (line: string) => void;
|
|
8
|
+
}
|
|
9
|
+
export interface RollbackCopy {
|
|
10
|
+
path: string;
|
|
11
|
+
from: number;
|
|
12
|
+
to: number;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Outside the migration's transaction (SQLite cannot attach inside one), committed on its own, so a migration that
|
|
16
|
+
* fails leaves it whole. Never blocks the migration: no room, or any failure, is a warning and no copy.
|
|
17
|
+
*/
|
|
18
|
+
export declare const takeRollbackCopy: (database: CacheDatabase, { from, to, tables, at }: {
|
|
19
|
+
from: number;
|
|
20
|
+
to: number;
|
|
21
|
+
tables: string[];
|
|
22
|
+
at: number;
|
|
23
|
+
}, { store, freeBytes, warn }: RollbackCopyOptions) => RollbackCopy | undefined;
|
|
24
|
+
/** Inside the migration's transaction: a migration that fails never records its copy, so cleanup never takes it. */
|
|
25
|
+
export declare const recordRollbackCopy: (database: CacheDatabase, copy: RollbackCopy, at: number) => void;
|
|
26
|
+
export declare const discardRollbackCopy: (copy: RollbackCopy) => void;
|
|
27
|
+
/**
|
|
28
|
+
* The day after: a copy recorded by a migration that committed, once the store is still at that version. Only the
|
|
29
|
+
* recorded file is deleted, never anything else in `backups/`.
|
|
30
|
+
*/
|
|
31
|
+
export declare const pruneRollbackCopies: (database: CacheDatabase, now: number) => void;
|
|
32
|
+
export interface RollbackCopyMeta {
|
|
33
|
+
from: number;
|
|
34
|
+
to: number;
|
|
35
|
+
tables: string[];
|
|
36
|
+
createdAt: number;
|
|
37
|
+
}
|
|
38
|
+
export interface RollbackResult extends RollbackCopyMeta {
|
|
39
|
+
/** The schema the store was at before the rollback. */
|
|
40
|
+
was: number;
|
|
41
|
+
restored: {
|
|
42
|
+
table: string;
|
|
43
|
+
rows: number;
|
|
44
|
+
}[];
|
|
45
|
+
rebuilt: string[];
|
|
46
|
+
foreignKeyViolations: number;
|
|
47
|
+
}
|
|
48
|
+
/** Puts a rollback copy back into the store, in place. The caller makes sure nothing else has the store open. */
|
|
49
|
+
export declare const rollBack: (database: CacheDatabase, path: string) => RollbackResult;
|
|
@@ -0,0 +1,290 @@
|
|
|
1
|
+
import { mkdirSync, rmSync, statfsSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { basename, dirname, join, parse } from "node:path";
|
|
3
|
+
import { CliError } from "@wirecat/cli-base";
|
|
4
|
+
/**
|
|
5
|
+
* A breaking migration (one that raises `min_compatible`) locks older tools out of the store. Before it runs, the
|
|
6
|
+
* base tables it names are copied into a small file in `backups/` beside the store, with the whole schema as it was;
|
|
7
|
+
* `store restore <copy>` puts them back and returns the file to the old version. Derived tables are never copied:
|
|
8
|
+
* search indexes, their queues, vectors, chunks, mentions and involvements are rebuilt from the base tables.
|
|
9
|
+
*/
|
|
10
|
+
const DERIVED = /(_fts|_words|_stems|_trigrams|_pending)$|^(embeddings|chunks|chunk_messages|mentions|involvements|search_terms)$|^vector_index/;
|
|
11
|
+
export const isDerivedTable = (name) => DERIVED.test(name);
|
|
12
|
+
/**
|
|
13
|
+
* Always copied, a few rows each: migrations seed rows into them, and the next upgrade after a rollback would insert
|
|
14
|
+
* those rows again and fail on its own unique keys.
|
|
15
|
+
*/
|
|
16
|
+
const BOOKKEEPING = ["store_settings", "search_index_state"];
|
|
17
|
+
const KEY = "rollback_copy:";
|
|
18
|
+
const SCHEMA = "rollback_copy";
|
|
19
|
+
const ROWID = "__rollback_rowid";
|
|
20
|
+
const DAY_MS = 24 * 60 * 60 * 1000;
|
|
21
|
+
// The migration's own write-ahead log needs room too.
|
|
22
|
+
const HEADROOM = 256 * 1024 * 1024;
|
|
23
|
+
const quote = (name) => `"${name.replaceAll('"', '""')}"`;
|
|
24
|
+
const megabytes = (bytes) => `${Math.ceil(bytes / 1024 / 1024)} MB`;
|
|
25
|
+
const freeBytesOf = (dir) => {
|
|
26
|
+
const { bavail, bsize } = statfsSync(dir);
|
|
27
|
+
return Number(bavail) * Number(bsize);
|
|
28
|
+
};
|
|
29
|
+
const tableIn = (database, schema, name) => database.prepare(`SELECT 1 FROM ${schema}.sqlite_master WHERE type = 'table' AND name = ?`).get(name) !== undefined;
|
|
30
|
+
const keepsRowid = (database, schema, table) => Number(database.prepare("SELECT wr FROM pragma_table_list WHERE schema = ? AND name = ?").get(schema, table)?.wr) ===
|
|
31
|
+
0;
|
|
32
|
+
const columnsOf = (database, schema, table) => database
|
|
33
|
+
.prepare("SELECT name FROM pragma_table_info(?, ?)")
|
|
34
|
+
.all(table, schema)
|
|
35
|
+
.map((row) => String(row.name));
|
|
36
|
+
const versionOf = (database) => Number(database.prepare("SELECT max(version) AS version FROM main.schema_migrations").get()?.version ?? 0);
|
|
37
|
+
const tableBytes = (database, tables) => {
|
|
38
|
+
try {
|
|
39
|
+
return Number(database
|
|
40
|
+
.prepare(`SELECT coalesce(sum(pgsize), 0) AS bytes FROM dbstat
|
|
41
|
+
WHERE schema = 'main' AND aggregate = 1 AND name IN (${tables.map(() => "?").join(", ")})`)
|
|
42
|
+
.get(...tables)?.bytes ?? 0);
|
|
43
|
+
}
|
|
44
|
+
catch {
|
|
45
|
+
// Bun's SQLite is built without dbstat: the whole file is the bound.
|
|
46
|
+
const pragma = (name) => Number(database.prepare(`PRAGMA ${name}`).get()?.[name] ?? 0);
|
|
47
|
+
return pragma("page_count") * pragma("page_size");
|
|
48
|
+
}
|
|
49
|
+
};
|
|
50
|
+
const stampOf = (at) => new Date(at).toISOString().replace(/[:.]/g, "-");
|
|
51
|
+
const attach = (database, path) => database.prepare(`ATTACH DATABASE ? AS ${SCHEMA}`).run(path);
|
|
52
|
+
/**
|
|
53
|
+
* Outside the migration's transaction (SQLite cannot attach inside one), committed on its own, so a migration that
|
|
54
|
+
* fails leaves it whole. Never blocks the migration: no room, or any failure, is a warning and no copy.
|
|
55
|
+
*/
|
|
56
|
+
export const takeRollbackCopy = (database, { from, to, tables, at }, { store, freeBytes = freeBytesOf, warn = (line) => process.stderr.write(`${line}\n`) }) => {
|
|
57
|
+
const dir = join(dirname(store), "backups");
|
|
58
|
+
const present = [...new Set([...BOOKKEEPING, ...tables])].filter((table) => tableIn(database, "main", table));
|
|
59
|
+
const afterwards = `going back to schema ${from} then needs an empty store, and its data pulled again from the sources`;
|
|
60
|
+
let path;
|
|
61
|
+
try {
|
|
62
|
+
mkdirSync(dir, { recursive: true, mode: 0o700 });
|
|
63
|
+
const needed = tableBytes(database, present);
|
|
64
|
+
const free = freeBytes(dir);
|
|
65
|
+
if (free < needed + HEADROOM) {
|
|
66
|
+
warn(`not enough free disk space for a rollback copy before schema ${to} (about ${megabytes(needed + HEADROOM)} ` +
|
|
67
|
+
`needed, ${megabytes(free)} free in ${dir}) — migrating without one; ${afterwards}`);
|
|
68
|
+
return undefined;
|
|
69
|
+
}
|
|
70
|
+
path = join(dir, `${parse(store).name}-rollback-schema-${from}-${stampOf(at)}.db`);
|
|
71
|
+
writeFileSync(path, "", { flag: "wx", mode: 0o600 });
|
|
72
|
+
attach(database, path);
|
|
73
|
+
try {
|
|
74
|
+
database.exec("BEGIN IMMEDIATE");
|
|
75
|
+
try {
|
|
76
|
+
const copied = versionOf(database);
|
|
77
|
+
database.exec(`CREATE TABLE ${SCHEMA}.rollback_meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)`);
|
|
78
|
+
database.exec(`CREATE TABLE ${SCHEMA}.rollback_schema AS
|
|
79
|
+
SELECT m.type, m.name, m.tbl_name, m.sql, coalesce(l.type, m.type) AS kind
|
|
80
|
+
FROM main.sqlite_master m LEFT JOIN pragma_table_list l ON l.schema = 'main' AND l.name = m.name
|
|
81
|
+
ORDER BY m.rowid`);
|
|
82
|
+
for (const table of present) {
|
|
83
|
+
const rowid = keepsRowid(database, "main", table) ? `rowid AS ${ROWID}, ` : "";
|
|
84
|
+
database.exec(`CREATE TABLE ${SCHEMA}.${quote(table)} AS SELECT ${rowid}* FROM main.${quote(table)}`);
|
|
85
|
+
}
|
|
86
|
+
const meta = { from: copied, to, tables: present, created_at: at };
|
|
87
|
+
const insert = database.prepare(`INSERT INTO ${SCHEMA}.rollback_meta (key, value) VALUES (?, ?)`);
|
|
88
|
+
for (const [key, value] of Object.entries(meta))
|
|
89
|
+
insert.run(key, JSON.stringify(value));
|
|
90
|
+
database.exec("COMMIT");
|
|
91
|
+
return { path, from: copied, to };
|
|
92
|
+
}
|
|
93
|
+
catch (error) {
|
|
94
|
+
database.exec("ROLLBACK");
|
|
95
|
+
throw error;
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
finally {
|
|
99
|
+
database.exec(`DETACH DATABASE ${SCHEMA}`);
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
catch (error) {
|
|
103
|
+
if (path)
|
|
104
|
+
rmSync(path, { force: true });
|
|
105
|
+
warn(`the rollback copy before schema ${to} could not be written (${error instanceof Error ? error.message : String(error)}) — migrating without one; ${afterwards}`);
|
|
106
|
+
return undefined;
|
|
107
|
+
}
|
|
108
|
+
};
|
|
109
|
+
/** Inside the migration's transaction: a migration that fails never records its copy, so cleanup never takes it. */
|
|
110
|
+
export const recordRollbackCopy = (database, copy, at) => {
|
|
111
|
+
database
|
|
112
|
+
.prepare("INSERT OR REPLACE INTO store_settings (key, value, at) VALUES (?, ?, ?)")
|
|
113
|
+
.run(`${KEY}${basename(copy.path)}`, JSON.stringify({ path: copy.path, from: copy.from, to: copy.to, at }), at);
|
|
114
|
+
};
|
|
115
|
+
export const discardRollbackCopy = (copy) => rmSync(copy.path, { force: true });
|
|
116
|
+
/**
|
|
117
|
+
* The day after: a copy recorded by a migration that committed, once the store is still at that version. Only the
|
|
118
|
+
* recorded file is deleted, never anything else in `backups/`.
|
|
119
|
+
*/
|
|
120
|
+
export const pruneRollbackCopies = (database, now) => {
|
|
121
|
+
const recorded = database
|
|
122
|
+
.prepare("SELECT key, value FROM store_settings WHERE key LIKE 'rollback\\_copy:%' ESCAPE '\\'")
|
|
123
|
+
.all();
|
|
124
|
+
if (recorded.length === 0)
|
|
125
|
+
return;
|
|
126
|
+
const version = versionOf(database);
|
|
127
|
+
for (const { key, value } of recorded) {
|
|
128
|
+
let copy;
|
|
129
|
+
try {
|
|
130
|
+
copy = JSON.parse(String(value));
|
|
131
|
+
}
|
|
132
|
+
catch {
|
|
133
|
+
continue;
|
|
134
|
+
}
|
|
135
|
+
if (typeof copy.path !== "string" || now - Number(copy.at) < DAY_MS || version < Number(copy.to))
|
|
136
|
+
continue;
|
|
137
|
+
if (!database.prepare("SELECT 1 FROM schema_migrations WHERE version = ?").get(Number(copy.to)))
|
|
138
|
+
continue;
|
|
139
|
+
rmSync(copy.path, { force: true });
|
|
140
|
+
database.prepare("DELETE FROM store_settings WHERE key = ?").run(String(key));
|
|
141
|
+
}
|
|
142
|
+
};
|
|
143
|
+
const objectsOf = (database, sql) => database
|
|
144
|
+
.prepare(sql)
|
|
145
|
+
.all()
|
|
146
|
+
.map((row) => ({
|
|
147
|
+
type: String(row.type),
|
|
148
|
+
name: String(row.name),
|
|
149
|
+
sql: row.sql === null ? null : String(row.sql),
|
|
150
|
+
kind: String(row.kind),
|
|
151
|
+
}));
|
|
152
|
+
const currentObjects = (database) => objectsOf(database, `SELECT m.type, m.name, m.sql, coalesce(l.type, m.type) AS kind
|
|
153
|
+
FROM main.sqlite_master m LEFT JOIN pragma_table_list l ON l.schema = 'main' AND l.name = m.name`);
|
|
154
|
+
const internal = ({ name, kind }) => name.startsWith("sqlite_") || kind === "shadow";
|
|
155
|
+
const metaOf = (database) => {
|
|
156
|
+
const values = new Map(database
|
|
157
|
+
.prepare(`SELECT key, value FROM ${SCHEMA}.rollback_meta`)
|
|
158
|
+
.all()
|
|
159
|
+
.map((row) => [String(row.key), JSON.parse(String(row.value))]));
|
|
160
|
+
const tables = values.get("tables");
|
|
161
|
+
return {
|
|
162
|
+
from: Number(values.get("from")),
|
|
163
|
+
to: Number(values.get("to")),
|
|
164
|
+
tables: Array.isArray(tables) ? tables.map(String) : [],
|
|
165
|
+
createdAt: Number(values.get("created_at")),
|
|
166
|
+
};
|
|
167
|
+
};
|
|
168
|
+
/** FTS5 tables over a table's own rows; the other search indexes hold their own text and wait for `store rebuild`. */
|
|
169
|
+
const contentOf = (sql) => /\bcontent\s*=\s*'([^']+)'/i.exec(sql ?? "")?.[1];
|
|
170
|
+
const RANK = { table: 0, index: 1, view: 2, trigger: 3 };
|
|
171
|
+
/**
|
|
172
|
+
* Returns the file to the schema the copy saved: every object made since is dropped, every table the migration
|
|
173
|
+
* changed is put back with its rows, a table only extended since is rebuilt to its old shape, and the history
|
|
174
|
+
* goes back to the copy's version. One transaction: on any failure the store is as it was.
|
|
175
|
+
*/
|
|
176
|
+
const rewind = (database, saved, named) => {
|
|
177
|
+
const before = new Map(saved.map((object) => [object.name, object]));
|
|
178
|
+
const same = (object) => before.get(object.name)?.type === object.type && before.get(object.name)?.sql === object.sql;
|
|
179
|
+
const touched = new Set(named);
|
|
180
|
+
const rebuilt = [];
|
|
181
|
+
// A trigger or view over a table dropped or rebuilt below would point nowhere; all of them come back from the copy.
|
|
182
|
+
for (const object of currentObjects(database)) {
|
|
183
|
+
if (object.type === "trigger")
|
|
184
|
+
database.exec(`DROP TRIGGER IF EXISTS ${quote(object.name)}`);
|
|
185
|
+
if (object.type === "view")
|
|
186
|
+
database.exec(`DROP VIEW IF EXISTS ${quote(object.name)}`);
|
|
187
|
+
if (object.type === "index" && object.sql !== null && !same(object))
|
|
188
|
+
database.exec(`DROP INDEX IF EXISTS ${quote(object.name)}`);
|
|
189
|
+
}
|
|
190
|
+
for (const object of currentObjects(database)) {
|
|
191
|
+
if (object.type !== "table" || internal(object))
|
|
192
|
+
continue;
|
|
193
|
+
const old = before.get(object.name);
|
|
194
|
+
if (named.has(object.name) || !old || (object.kind === "virtual" && !same(object))) {
|
|
195
|
+
database.exec(`DROP TABLE ${quote(object.name)}`);
|
|
196
|
+
}
|
|
197
|
+
else if (!same(object) && old.sql) {
|
|
198
|
+
rebuild(database, object.name, old.sql);
|
|
199
|
+
rebuilt.push(object.name);
|
|
200
|
+
touched.add(object.name);
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
const present = new Set(currentObjects(database).map(({ name }) => name));
|
|
204
|
+
const missing = saved
|
|
205
|
+
.filter((object) => !internal(object) && object.sql !== null && !present.has(object.name))
|
|
206
|
+
.sort((a, b) => (RANK[a.type] ?? 9) - (RANK[b.type] ?? 9));
|
|
207
|
+
for (const object of missing) {
|
|
208
|
+
database.exec(object.sql);
|
|
209
|
+
if (object.type === "table" && named.has(object.name))
|
|
210
|
+
restoreRows(database, object.name);
|
|
211
|
+
if (object.kind === "virtual")
|
|
212
|
+
touched.add(object.name);
|
|
213
|
+
}
|
|
214
|
+
for (const object of saved) {
|
|
215
|
+
const content = contentOf(object.sql);
|
|
216
|
+
if (object.kind !== "virtual" || !(content && (touched.has(content) || touched.has(object.name))))
|
|
217
|
+
continue;
|
|
218
|
+
database.exec(`INSERT INTO ${quote(object.name)} (${quote(object.name)}) VALUES ('rebuild')`);
|
|
219
|
+
}
|
|
220
|
+
return rebuilt;
|
|
221
|
+
};
|
|
222
|
+
const rebuild = (database, table, sql) => {
|
|
223
|
+
const aside = `__rollback_${table}`;
|
|
224
|
+
database.exec(`ALTER TABLE ${quote(table)} RENAME TO ${quote(aside)}`);
|
|
225
|
+
database.exec(sql);
|
|
226
|
+
const kept = new Set(columnsOf(database, "main", aside));
|
|
227
|
+
const columns = columnsOf(database, "main", table).filter((column) => kept.has(column));
|
|
228
|
+
const rowid = keepsRowid(database, "main", table) ? ["rowid"] : [];
|
|
229
|
+
const list = [...rowid, ...columns.map(quote)].join(", ");
|
|
230
|
+
database.exec(`INSERT INTO ${quote(table)} (${list}) SELECT ${list} FROM ${quote(aside)}`);
|
|
231
|
+
database.exec(`DROP TABLE ${quote(aside)}`);
|
|
232
|
+
};
|
|
233
|
+
const restoreRows = (database, table) => {
|
|
234
|
+
const copied = new Set(columnsOf(database, SCHEMA, table));
|
|
235
|
+
const columns = columnsOf(database, "main", table).filter((column) => copied.has(column));
|
|
236
|
+
const rowid = keepsRowid(database, "main", table) && copied.has(ROWID);
|
|
237
|
+
const into = [...(rowid ? ["rowid"] : []), ...columns.map(quote)].join(", ");
|
|
238
|
+
const from = [...(rowid ? [ROWID] : []), ...columns.map(quote)].join(", ");
|
|
239
|
+
database.exec(`INSERT INTO main.${quote(table)} (${into}) SELECT ${from} FROM ${SCHEMA}.${quote(table)}`);
|
|
240
|
+
};
|
|
241
|
+
/** Puts a rollback copy back into the store, in place. The caller makes sure nothing else has the store open. */
|
|
242
|
+
export const rollBack = (database, path) => {
|
|
243
|
+
attach(database, path);
|
|
244
|
+
let result;
|
|
245
|
+
try {
|
|
246
|
+
if (!tableIn(database, SCHEMA, "rollback_meta"))
|
|
247
|
+
throw new CliError("validation_error", `${path} is not a rollback copy`);
|
|
248
|
+
const meta = metaOf(database);
|
|
249
|
+
const was = versionOf(database);
|
|
250
|
+
if (!database.prepare("SELECT 1 FROM main.schema_migrations WHERE version = ?").get(meta.to)) {
|
|
251
|
+
throw new CliError("validation_error", `the store is at schema ${was} and was never migrated to ${meta.to}, the schema this copy was taken before — nothing to roll back`);
|
|
252
|
+
}
|
|
253
|
+
const saved = objectsOf(database, `SELECT type, name, sql, kind FROM ${SCHEMA}.rollback_schema ORDER BY rowid`);
|
|
254
|
+
database.exec("PRAGMA foreign_keys = OFF");
|
|
255
|
+
// Renaming a table aside must not rewrite the references other tables hold to it.
|
|
256
|
+
database.exec("PRAGMA legacy_alter_table = ON");
|
|
257
|
+
try {
|
|
258
|
+
database.exec("BEGIN IMMEDIATE");
|
|
259
|
+
try {
|
|
260
|
+
const named = new Set(meta.tables);
|
|
261
|
+
const rebuilt = rewind(database, saved, named);
|
|
262
|
+
database.prepare("DELETE FROM main.schema_migrations WHERE version > ?").run(meta.from);
|
|
263
|
+
database.prepare("DELETE FROM main.store_settings WHERE key = ?").run(`${KEY}${basename(path)}`);
|
|
264
|
+
database.exec("COMMIT");
|
|
265
|
+
const count = (table) => Number(database.prepare(`SELECT count(*) AS n FROM main.${quote(table)}`).get()?.n ?? 0);
|
|
266
|
+
result = {
|
|
267
|
+
...meta,
|
|
268
|
+
was,
|
|
269
|
+
restored: meta.tables.map((table) => ({ table, rows: count(table) })),
|
|
270
|
+
rebuilt,
|
|
271
|
+
};
|
|
272
|
+
}
|
|
273
|
+
catch (error) {
|
|
274
|
+
database.exec("ROLLBACK");
|
|
275
|
+
throw error;
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
finally {
|
|
279
|
+
database.exec("PRAGMA legacy_alter_table = OFF");
|
|
280
|
+
database.exec("PRAGMA foreign_keys = ON");
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
finally {
|
|
284
|
+
database.exec(`DETACH DATABASE ${SCHEMA}`);
|
|
285
|
+
}
|
|
286
|
+
// Only the tables put back: a check of the whole store reads every row of it.
|
|
287
|
+
const checked = [...result.restored.map(({ table }) => table), ...result.rebuilt];
|
|
288
|
+
const foreignKeyViolations = checked.reduce((total, table) => total + database.prepare(`PRAGMA foreign_key_check(${quote(table)})`).all().length, 0);
|
|
289
|
+
return { ...result, foreignKeyViolations };
|
|
290
|
+
};
|
package/dist/store/store.d.ts
CHANGED
|
@@ -133,6 +133,7 @@ export interface MessageStore {
|
|
|
133
133
|
export type { ChatHistoryShape, StoredHistorySkip } from "./sqlite/history-skips.js";
|
|
134
134
|
export { BACKFILL_ON_OPEN, type StoreOptions } from "./sqlite/open-store.js";
|
|
135
135
|
export type { ChatScope, NewestMessage, ReplyDelay, WaitingChat } from "./sqlite/reply-times.js";
|
|
136
|
+
export { rollBack } from "./sqlite/rollback-copy.js";
|
|
136
137
|
export { fileFacts, readStoreTables, type TableKind } from "./sqlite/store-tables.js";
|
|
137
138
|
export declare const openStore: (options?: StoreOptions) => Promise<MessageStore>;
|
|
138
139
|
export * from "./exports.js";
|
package/dist/store/store.js
CHANGED
|
@@ -50,6 +50,7 @@ import { syncStoreOver } from "./sync.js";
|
|
|
50
50
|
import { tagsStoreOver } from "./tags.js";
|
|
51
51
|
import { accountVectorsOver, vectorsStoreOver } from "./vectors.js";
|
|
52
52
|
export { BACKFILL_ON_OPEN } from "./sqlite/open-store.js";
|
|
53
|
+
export { rollBack } from "./sqlite/rollback-copy.js";
|
|
53
54
|
export { fileFacts, readStoreTables } from "./sqlite/store-tables.js";
|
|
54
55
|
export const openStore = async (options = {}) => {
|
|
55
56
|
const context = await openStoreContext(options);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@wirecat/cli-core",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.596.0",
|
|
4
4
|
"description": "The messenger-neutral half of a messaging CLI: domain model, message locators, rendering, name resolution, the SQLite seam, the command skeleton and shared read commands, the send guard, run records and the message store",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "Viacheslav Ptsarev",
|
|
@@ -13,6 +13,12 @@
|
|
|
13
13
|
"url": "https://github.com/WireCatLabs/cli-core/issues"
|
|
14
14
|
},
|
|
15
15
|
"type": "module",
|
|
16
|
+
"wirecat": {
|
|
17
|
+
"store": {
|
|
18
|
+
"speaks": 53,
|
|
19
|
+
"minCompatible": 34
|
|
20
|
+
}
|
|
21
|
+
},
|
|
16
22
|
"engines": {
|
|
17
23
|
"node": "^22.16.0 || >=24"
|
|
18
24
|
},
|