@hraness/sponge 0.2.6 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -9
- package/bin/sponge.js +8 -7
- package/dist/contracts/index.js +24 -6
- package/dist/contracts/validate.js +106 -29
- package/dist/migrations/008.d.ts +3 -0
- package/dist/migrations/008.js +5 -0
- package/dist/migrations/009.d.ts +3 -0
- package/dist/migrations/009.js +8 -0
- package/dist/migrations/010.d.ts +3 -0
- package/dist/migrations/010.js +11 -0
- package/dist/migrations/index.d.ts +15 -0
- package/dist/migrations/index.js +39 -0
- package/dist/src/acquisition/capture.js +87 -28
- package/dist/src/acquisition/cleanup.d.ts +0 -4
- package/dist/src/acquisition/cleanup.js +3 -5
- package/dist/src/acquisition/errors.js +17 -2
- package/dist/src/acquisition/files.d.ts +0 -1
- package/dist/src/acquisition/files.js +14 -11
- package/dist/src/acquisition/index.d.ts +2 -3
- package/dist/src/acquisition/index.js +1 -2
- package/dist/src/acquisition/lock.js +1 -1
- package/dist/src/acquisition/normalize.js +57 -18
- package/dist/src/acquisition/pins.js +22 -4
- package/dist/src/acquisition/process.js +67 -29
- package/dist/src/acquisition/runtime.d.ts +5 -7
- package/dist/src/acquisition/runtime.js +129 -68
- package/dist/src/acquisition/verify-helper.js +10 -3
- package/dist/src/annotations/quote.js +40 -12
- package/dist/src/annotations/store.d.ts +0 -1
- package/dist/src/annotations/store.js +66 -14
- package/dist/src/app/article.d.ts +2 -1
- package/dist/src/app/article.js +66 -70
- package/dist/src/app/bundle-files.d.ts +0 -4
- package/dist/src/app/bundle-files.js +11 -14
- package/dist/src/app/markdown.js +18 -6
- package/dist/src/app/source.d.ts +0 -4
- package/dist/src/app/source.js +18 -14
- package/dist/src/cli/arguments.d.ts +2 -8
- package/dist/src/cli/arguments.js +67 -80
- package/dist/src/cli/commands.d.ts +42 -0
- package/dist/src/cli/commands.js +795 -0
- package/dist/src/cli/config.js +12 -9
- package/dist/src/cli/digest.d.ts +1 -1
- package/dist/src/cli/digest.js +38 -16
- package/dist/src/cli/import.js +25 -4
- package/dist/src/cli/indexing.d.ts +2 -0
- package/dist/src/cli/indexing.js +26 -9
- package/dist/src/cli/main.js +47 -386
- package/dist/src/cli/portability.d.ts +0 -2
- package/dist/src/cli/portability.js +13 -8
- package/dist/src/cli/queue.js +12 -8
- package/dist/src/cli/runtimes.d.ts +1 -3
- package/dist/src/cli/runtimes.js +13 -6
- package/dist/src/cli/terminal.js +1 -1
- package/dist/src/cli/worker.d.ts +7 -0
- package/dist/src/cli/worker.js +54 -15
- package/dist/src/derivations/kinds.d.ts +34 -0
- package/dist/src/derivations/kinds.js +14 -0
- package/dist/src/derivations/types.d.ts +48 -0
- package/dist/src/derivations/types.js +1 -0
- package/dist/src/digest/derivation.d.ts +25 -0
- package/dist/src/digest/derivation.js +124 -0
- package/dist/src/digest/editorial.d.ts +9 -46
- package/dist/src/digest/editorial.js +57 -49
- package/dist/src/digest/endpoint.js +63 -27
- package/dist/src/digest/pipeline.d.ts +49 -28
- package/dist/src/digest/pipeline.js +343 -100
- package/dist/src/digest/schema.d.ts +11 -25
- package/dist/src/digest/schema.js +86 -18
- package/dist/src/errors.d.ts +46 -0
- package/dist/src/errors.js +117 -0
- package/dist/src/identity/query-policy.d.ts +0 -5
- package/dist/src/identity/query-policy.js +61 -9
- package/dist/src/identity/uri.d.ts +6 -5
- package/dist/src/identity/uri.js +37 -18
- package/dist/src/identity/visibility.d.ts +11 -1
- package/dist/src/identity/visibility.js +81 -17
- package/dist/src/importers/apply.js +104 -36
- package/dist/src/importers/custody.js +27 -9
- package/dist/src/importers/inspect.js +125 -38
- package/dist/src/importers/jungle-digest.d.ts +3 -6
- package/dist/src/importers/jungle-digest.js +196 -69
- package/dist/src/importers/jungle-store.d.ts +15 -6
- package/dist/src/importers/jungle-store.js +122 -28
- package/dist/src/importers/jungle.d.ts +14 -14
- package/dist/src/importers/jungle.js +136 -31
- package/dist/src/importers/manifest.js +57 -21
- package/dist/src/importers/verify-helper.js +18 -4
- package/dist/src/indexing/image.js +29 -10
- package/dist/src/indexing/index.d.ts +37 -6
- package/dist/src/indexing/index.js +360 -124
- package/dist/src/indexing/jobs.js +156 -47
- package/dist/src/indexing/prune-run.d.ts +2 -0
- package/dist/src/indexing/prune-run.js +69 -26
- package/dist/src/indexing/prune.d.ts +12 -3
- package/dist/src/indexing/prune.js +85 -35
- package/dist/src/indexing/runner.d.ts +2 -1
- package/dist/src/indexing/runner.js +83 -40
- package/dist/src/indexing/upgrade.d.ts +9 -0
- package/dist/src/indexing/upgrade.js +79 -0
- package/dist/src/inference/asr/bridge.js +6 -2
- package/dist/src/inference/asr/protocol.js +109 -19
- package/dist/src/inference/asr/runtime.js +58 -21
- package/dist/src/inference/asr/windows.js +9 -5
- package/dist/src/inference/embedding/index.d.ts +2 -6
- package/dist/src/inference/embedding/index.js +89 -29
- package/dist/src/ingest/attempt.d.ts +29 -0
- package/dist/src/ingest/attempt.js +55 -0
- package/dist/src/media/capture-helper.js +7 -2
- package/dist/src/media/capture.d.ts +2 -1
- package/dist/src/media/capture.js +124 -43
- package/dist/src/media/handler.d.ts +4 -2
- package/dist/src/media/handler.js +88 -70
- package/dist/src/media/transcribe.js +63 -21
- package/dist/src/metadata/article.js +49 -17
- package/dist/src/models/artifacts.d.ts +10 -8
- package/dist/src/models/artifacts.js +237 -106
- package/dist/src/models/catalog.d.ts +2 -5
- package/dist/src/models/catalog.js +94 -25
- package/dist/src/models/download.js +55 -15
- package/dist/src/models/runtime.d.ts +0 -2
- package/dist/src/models/runtime.js +69 -33
- package/dist/src/pdf/capture-helper.js +3 -1
- package/dist/src/pdf/capture.d.ts +2 -1
- package/dist/src/pdf/capture.js +95 -27
- package/dist/src/pdf/errors.js +12 -1
- package/dist/src/pdf/handler.d.ts +2 -1
- package/dist/src/pdf/handler.js +70 -71
- package/dist/src/pdf/index.d.ts +1 -3
- package/dist/src/pdf/index.js +0 -2
- package/dist/src/pdf/input.js +12 -7
- package/dist/src/pdf/normalize.js +80 -21
- package/dist/src/pdf/source-identity.js +3 -2
- package/dist/src/pdf/source.js +12 -8
- package/dist/src/pdf/verify-helper.js +15 -3
- package/dist/src/portability/backup.d.ts +2 -9
- package/dist/src/portability/backup.js +106 -22
- package/dist/src/portability/database.js +51 -17
- package/dist/src/portability/export.js +35 -9
- package/dist/src/portability/files.d.ts +1 -4
- package/dist/src/portability/files.js +127 -67
- package/dist/src/portability/gc.js +65 -22
- package/dist/src/publish/component.js +1 -1
- package/dist/src/publish/feed.d.ts +2 -2
- package/dist/src/publish/feed.js +75 -29
- package/dist/src/publish/identity.d.ts +5 -1
- package/dist/src/publish/identity.js +31 -11
- package/dist/src/publish/index.d.ts +2 -3
- package/dist/src/publish/index.js +34 -21
- package/dist/src/publish/projection.d.ts +2 -9
- package/dist/src/publish/projection.js +21 -14
- package/dist/src/search/extension.js +3 -1
- package/dist/src/search/hybrid.d.ts +3 -8
- package/dist/src/search/hybrid.js +45 -11
- package/dist/src/search/lexical.js +18 -4
- package/dist/src/search/query.js +56 -25
- package/dist/src/search/vector.js +37 -10
- package/dist/src/service/launch-agent.d.ts +5 -3
- package/dist/src/service/launch-agent.js +272 -83
- package/dist/src/service/lifetime.js +47 -14
- package/dist/src/service/loop.d.ts +3 -3
- package/dist/src/service/loop.js +47 -19
- package/dist/src/store/annotations.d.ts +32 -0
- package/dist/src/store/annotations.js +44 -0
- package/dist/src/store/backups.d.ts +18 -0
- package/dist/src/store/backups.js +110 -0
- package/dist/src/store/blobs.d.ts +24 -0
- package/dist/src/store/blobs.js +139 -0
- package/dist/src/store/changes.d.ts +5 -12
- package/dist/src/store/changes.js +108 -56
- package/dist/src/store/context.d.ts +15 -0
- package/dist/src/store/context.js +19 -0
- package/dist/src/store/derivations.d.ts +42 -0
- package/dist/src/store/derivations.js +104 -0
- package/dist/src/store/files.d.ts +0 -2
- package/dist/src/store/files.js +19 -24
- package/dist/src/store/ids.d.ts +30 -0
- package/dist/src/store/ids.js +34 -0
- package/dist/src/store/imports.d.ts +26 -0
- package/dist/src/store/imports.js +57 -0
- package/dist/src/store/index.d.ts +3 -186
- package/dist/src/store/index.js +2 -809
- package/dist/src/store/lease.d.ts +20 -0
- package/dist/src/store/lease.js +62 -0
- package/dist/src/store/library.d.ts +75 -0
- package/dist/src/store/library.js +174 -0
- package/dist/src/store/open.d.ts +7 -0
- package/dist/src/store/open.js +183 -0
- package/dist/src/store/queue.d.ts +65 -0
- package/dist/src/store/queue.js +327 -0
- package/dist/src/store/revisions.d.ts +12 -0
- package/dist/src/store/revisions.js +189 -0
- package/dist/src/store/search-index.d.ts +63 -0
- package/dist/src/store/search-index.js +190 -0
- package/dist/src/store/sync-ready.d.ts +6 -0
- package/dist/src/store/sync-ready.js +121 -0
- package/dist/src/store/types.d.ts +1 -5
- package/dist/src/store/types.js +1 -8
- package/dist/src/util/canonical.d.ts +2 -0
- package/dist/src/util/canonical.js +17 -0
- package/dist/src/util/exact.d.ts +2 -0
- package/dist/src/util/exact.js +9 -0
- package/dist/src/util/fsync.d.ts +4 -0
- package/dist/src/util/fsync.js +22 -0
- package/dist/src/util/hash.d.ts +4 -0
- package/dist/src/util/hash.js +8 -0
- package/dist/src/worker/basic.d.ts +0 -1
- package/dist/src/worker/basic.js +23 -7
- package/dist/src/worker/custody.d.ts +4 -1
- package/dist/src/worker/custody.js +111 -38
- package/dist/src/worker/engine.d.ts +2 -1
- package/dist/src/worker/engine.js +42 -20
- package/dist/src/worker/identity.js +13 -5
- package/dist/src/worker/journal.d.ts +1 -1
- package/dist/src/worker/journal.js +54 -24
- package/dist/src/worker/lock.d.ts +18 -0
- package/dist/src/worker/lock.js +196 -0
- package/dist/src/worker/runtime-gc.d.ts +4 -3
- package/dist/src/worker/runtime-gc.js +68 -41
- package/dist/src/worker/supervisor-child.js +19 -6
- package/dist/src/worker/supervisor.js +41 -8
- package/native/runtime-manifest.json +4 -4
- package/package.json +10 -2
- package/dist/src/digest/store.d.ts +0 -57
- package/dist/src/digest/store.js +0 -79
- package/dist/src/store/foundation.d.ts +0 -7
- package/dist/src/store/foundation.js +0 -7
|
@@ -1,41 +1,95 @@
|
|
|
1
|
-
import { constants, openSync, closeSync, fstatSync, fsyncSync, unlinkSync, rmSync } from 'node:fs';
|
|
1
|
+
import { constants, openSync, closeSync, fstatSync, fsyncSync, unlinkSync, rmSync, chmodSync, fchmodSync, } from 'node:fs';
|
|
2
2
|
import { dirname, join, resolve } from 'node:path';
|
|
3
3
|
import { openLibrary } from '../store/index.js';
|
|
4
|
-
import { safeDirectory } from '../store/files.js';
|
|
5
|
-
import {
|
|
4
|
+
import { safeDirectory, pathExists } from '../store/files.js';
|
|
5
|
+
import { loadPublishKey, publishKeyPath, maximumPublishKeyBytes } from '../publish/identity.js';
|
|
6
6
|
import { inspectDatabase, neutralizeRestoredDatabase } from './database.js';
|
|
7
|
-
import { bounded, exact, fileDescriptor, invalid, maximumDatabaseBytes, maximumEntries, maximumTotalBytes, openRead, readExact, transfer, writeAll, describe, newDirectory, discardOwned,
|
|
7
|
+
import { bounded, digestString, exact, fileDescriptor, invalid, maximumDatabaseBytes, maximumEntries, maximumTotalBytes, openRead, readExact, transfer, writeAll, describe, newDirectory, discardOwned, writePrivateExclusive, } from './files.js';
|
|
8
|
+
import { syncDirectory } from '../util/fsync.js';
|
|
9
|
+
import { SpongeError } from '../errors.js';
|
|
8
10
|
const magic = Buffer.from('SPONGE-BACKUP-1\n');
|
|
9
11
|
const maxManifestBytes = 32 * 1024 ** 2;
|
|
12
|
+
function publishKeyEntry(value) {
|
|
13
|
+
if (value === null)
|
|
14
|
+
return null;
|
|
15
|
+
const r = exact(value, ['path', 'bytes', 'sha256']);
|
|
16
|
+
if (r.path !== publishKeyPath)
|
|
17
|
+
invalid();
|
|
18
|
+
return { path: publishKeyPath, bytes: bounded(r.bytes, maximumPublishKeyBytes), sha256: digestString(r.sha256) };
|
|
19
|
+
}
|
|
10
20
|
function manifest(value) {
|
|
11
|
-
const
|
|
12
|
-
|
|
21
|
+
const v = value && typeof value === 'object' ? value.version : undefined;
|
|
22
|
+
const r = exact(value, [
|
|
23
|
+
'format',
|
|
24
|
+
'version',
|
|
25
|
+
'schemaVersion',
|
|
26
|
+
'libraryId',
|
|
27
|
+
'revisionEpoch',
|
|
28
|
+
'files',
|
|
29
|
+
...(v === 2 ? ['publishKey'] : []),
|
|
30
|
+
]);
|
|
31
|
+
if (r.format !== 'sponge-backup' ||
|
|
32
|
+
(r.version !== 1 && r.version !== 2) ||
|
|
33
|
+
typeof r.libraryId !== 'string' ||
|
|
34
|
+
r.libraryId.length > 256 ||
|
|
35
|
+
!Array.isArray(r.files) ||
|
|
36
|
+
r.files.length < 1 ||
|
|
37
|
+
r.files.length > maximumEntries + 1)
|
|
13
38
|
invalid();
|
|
14
39
|
const files = r.files.map(fileDescriptor);
|
|
15
|
-
if (files[0]?.path !== 'library.sqlite' ||
|
|
40
|
+
if (files[0]?.path !== 'library.sqlite' ||
|
|
41
|
+
files.slice(1).some((f) => f.path === 'library.sqlite') ||
|
|
42
|
+
new Set(files.map((f) => f.path)).size !== files.length)
|
|
16
43
|
invalid();
|
|
17
|
-
return {
|
|
44
|
+
return {
|
|
45
|
+
format: 'sponge-backup',
|
|
46
|
+
version: r.version,
|
|
47
|
+
schemaVersion: bounded(r.schemaVersion, Number.MAX_SAFE_INTEGER),
|
|
48
|
+
libraryId: r.libraryId,
|
|
49
|
+
revisionEpoch: bounded(r.revisionEpoch, Number.MAX_SAFE_INTEGER),
|
|
50
|
+
files,
|
|
51
|
+
publishKey: r.version === 2 ? publishKeyEntry(r.publishKey) : null,
|
|
52
|
+
};
|
|
18
53
|
}
|
|
19
54
|
function sameSnapshot(info, m) {
|
|
20
|
-
if (info.schemaVersion !== m.schemaVersion ||
|
|
55
|
+
if (info.schemaVersion !== m.schemaVersion ||
|
|
56
|
+
info.libraryId !== m.libraryId ||
|
|
57
|
+
info.revisionEpoch !== m.revisionEpoch)
|
|
21
58
|
invalid();
|
|
22
|
-
const expected = info.blobs.map(b => ({ path: b.relativePath, bytes: b.bytes, sha256: b.sha256 }));
|
|
59
|
+
const expected = info.blobs.map((b) => ({ path: b.relativePath, bytes: b.bytes, sha256: b.sha256 }));
|
|
23
60
|
if (JSON.stringify(expected) !== JSON.stringify(m.files.slice(1)))
|
|
24
61
|
invalid();
|
|
25
62
|
}
|
|
26
63
|
export async function backupLibrary(store, output) {
|
|
27
64
|
if (resolve(output) !== output)
|
|
28
|
-
throw new
|
|
65
|
+
throw new SpongeError('invalid-input');
|
|
29
66
|
safeDirectory(dirname(output));
|
|
30
|
-
const snapshot = await store.
|
|
67
|
+
const snapshot = await store.backups.snapshot();
|
|
31
68
|
let fd, created = false;
|
|
32
69
|
try {
|
|
33
70
|
const info = inspectDatabase(snapshot.path);
|
|
34
|
-
const files = [
|
|
35
|
-
|
|
71
|
+
const files = [
|
|
72
|
+
describe(snapshot.path, 'library.sqlite', maximumDatabaseBytes),
|
|
73
|
+
...info.blobs.map((b) => ({ path: b.relativePath, bytes: b.bytes, sha256: b.sha256 })),
|
|
74
|
+
];
|
|
75
|
+
// The publishing key is immutable once linked into place; a key that would not load is refused here, not discovered at restore.
|
|
76
|
+
const keyPath = join(store.root, publishKeyPath), publishKey = pathExists(keyPath)
|
|
77
|
+
? (loadPublishKey(keyPath), describe(keyPath, publishKeyPath, maximumPublishKeyBytes))
|
|
78
|
+
: null;
|
|
79
|
+
const m = {
|
|
80
|
+
format: 'sponge-backup',
|
|
81
|
+
version: 2,
|
|
82
|
+
schemaVersion: info.schemaVersion,
|
|
83
|
+
libraryId: info.libraryId,
|
|
84
|
+
revisionEpoch: info.revisionEpoch,
|
|
85
|
+
files,
|
|
86
|
+
publishKey,
|
|
87
|
+
};
|
|
36
88
|
const json = Buffer.from(JSON.stringify(m));
|
|
37
|
-
if (json.length > maxManifestBytes ||
|
|
38
|
-
|
|
89
|
+
if (json.length > maxManifestBytes ||
|
|
90
|
+
[...files, ...(publishKey ? [publishKey] : [])].reduce((n, f) => n + f.bytes, json.length + magic.length + 4) >
|
|
91
|
+
maximumTotalBytes)
|
|
92
|
+
throw new SpongeError('limit-exceeded');
|
|
39
93
|
fd = openSync(output, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600);
|
|
40
94
|
created = true;
|
|
41
95
|
writeAll(fd, magic);
|
|
@@ -43,7 +97,7 @@ export async function backupLibrary(store, output) {
|
|
|
43
97
|
length.writeUInt32BE(json.length);
|
|
44
98
|
writeAll(fd, length);
|
|
45
99
|
writeAll(fd, json);
|
|
46
|
-
for (const file of files) {
|
|
100
|
+
for (const file of publishKey ? [...files, publishKey] : files) {
|
|
47
101
|
const input = openRead(file.path === 'library.sqlite' ? snapshot.path : join(store.root, file.path));
|
|
48
102
|
try {
|
|
49
103
|
if (fstatSync(input).size !== file.bytes || transfer(input, file.bytes, fd) !== file.sha256)
|
|
@@ -58,7 +112,13 @@ export async function backupLibrary(store, output) {
|
|
|
58
112
|
fd = undefined;
|
|
59
113
|
syncDirectory(dirname(output));
|
|
60
114
|
const result = describe(output, 'archive', maximumTotalBytes);
|
|
61
|
-
return {
|
|
115
|
+
return {
|
|
116
|
+
output,
|
|
117
|
+
sha256: result.sha256,
|
|
118
|
+
bytes: result.bytes,
|
|
119
|
+
itemsSnapshotEpoch: info.revisionEpoch,
|
|
120
|
+
publishKey: publishKey !== null,
|
|
121
|
+
};
|
|
62
122
|
}
|
|
63
123
|
catch (error) {
|
|
64
124
|
if (fd !== undefined) {
|
|
@@ -78,7 +138,7 @@ export async function backupLibrary(store, output) {
|
|
|
78
138
|
rmSync(snapshot.path + side, { force: true });
|
|
79
139
|
}
|
|
80
140
|
finally {
|
|
81
|
-
store.
|
|
141
|
+
store.backups.finish(snapshot.id);
|
|
82
142
|
}
|
|
83
143
|
}
|
|
84
144
|
}
|
|
@@ -101,7 +161,8 @@ export async function restoreLibrary(input, into) {
|
|
|
101
161
|
invalid();
|
|
102
162
|
}
|
|
103
163
|
const m = manifest(parsed);
|
|
104
|
-
if (m.files.reduce((n, f) => n + f.bytes, magic.length + 4 + length) !==
|
|
164
|
+
if ([...m.files, ...(m.publishKey ? [m.publishKey] : [])].reduce((n, f) => n + f.bytes, magic.length + 4 + length) !==
|
|
165
|
+
size)
|
|
105
166
|
invalid();
|
|
106
167
|
target = newDirectory(into);
|
|
107
168
|
for (const file of m.files) {
|
|
@@ -118,16 +179,39 @@ export async function restoreLibrary(input, into) {
|
|
|
118
179
|
closeSync(out);
|
|
119
180
|
}
|
|
120
181
|
}
|
|
182
|
+
// The key goes only into the new directory created above (exclusive create), so restore never replaces an existing key.
|
|
183
|
+
if (m.publishKey) {
|
|
184
|
+
const dir = safeDirectory(join(target.root, 'keys'), true);
|
|
185
|
+
chmodSync(dir, 0o700);
|
|
186
|
+
const path = join(target.root, publishKeyPath), out = openSync(path, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600);
|
|
187
|
+
try {
|
|
188
|
+
if (transfer(fd, m.publishKey.bytes, out) !== m.publishKey.sha256)
|
|
189
|
+
invalid();
|
|
190
|
+
fchmodSync(out, 0o600);
|
|
191
|
+
fsyncSync(out);
|
|
192
|
+
}
|
|
193
|
+
finally {
|
|
194
|
+
closeSync(out);
|
|
195
|
+
}
|
|
196
|
+
syncDirectory(dir);
|
|
197
|
+
loadPublishKey(path);
|
|
198
|
+
}
|
|
121
199
|
sameSnapshot(inspectDatabase(join(into, 'library.sqlite')), m);
|
|
122
200
|
const state = neutralizeRestoredDatabase(join(into, 'library.sqlite'));
|
|
123
201
|
writePrivateExclusive(join(into, '.sponge-library'), 'sponge-library-v1\n');
|
|
124
202
|
// Mark bindings as deliberately absent. This never installs runtime/auth/model files.
|
|
125
203
|
safeDirectory(join(into, 'runtime'), true);
|
|
126
|
-
writePrivateExclusive(join(into, 'runtime', 'restore.json'), JSON.stringify({
|
|
204
|
+
writePrivateExclusive(join(into, 'runtime', 'restore.json'), JSON.stringify({
|
|
205
|
+
schemaVersion: 1,
|
|
206
|
+
libraryId: m.libraryId,
|
|
207
|
+
ownerIncarnation: state.incarnation,
|
|
208
|
+
connections: 'unresolved',
|
|
209
|
+
heldJobs: state.heldJobs,
|
|
210
|
+
}) + '\n');
|
|
127
211
|
const store = await openLibrary(into);
|
|
128
212
|
store.close();
|
|
129
213
|
syncDirectory(into);
|
|
130
|
-
return { root: into, libraryId: m.libraryId, ...state };
|
|
214
|
+
return { root: into, libraryId: m.libraryId, ...state, publishKey: m.publishKey !== null };
|
|
131
215
|
}
|
|
132
216
|
catch (error) {
|
|
133
217
|
if (target)
|
|
@@ -1,46 +1,78 @@
|
|
|
1
1
|
import { DatabaseSync } from 'node:sqlite';
|
|
2
2
|
import { randomUUID } from 'node:crypto';
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
3
|
+
import { migrations } from '../../migrations/index.js';
|
|
4
|
+
import { regularFile } from '../store/files.js';
|
|
5
|
+
import { sha256 } from '../util/hash.js';
|
|
6
6
|
import { blobPath, bounded, digestString, invalid, maximumBlobBytes, maximumEntries } from './files.js';
|
|
7
|
-
|
|
7
|
+
import { SpongeError } from '../errors.js';
|
|
8
|
+
function schema(db) {
|
|
9
|
+
return JSON.stringify(db.prepare('SELECT type,name,tbl_name,sql FROM sqlite_schema ORDER BY type,name').all());
|
|
10
|
+
}
|
|
8
11
|
/** Compare executable schema before any restored SQL mutation, not just claimed migration hashes. */
|
|
9
12
|
export function inspectDatabase(path) {
|
|
10
13
|
regularFile(path);
|
|
14
|
+
// SQLite's own errors on a forged or corrupt file (not a database, malformed image) are an integrity failure, not a crash.
|
|
15
|
+
try {
|
|
16
|
+
return inspectOpened(path);
|
|
17
|
+
}
|
|
18
|
+
catch (error) {
|
|
19
|
+
if (error instanceof SpongeError)
|
|
20
|
+
throw error;
|
|
21
|
+
throw new SpongeError('integrity', { cause: error });
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
function inspectOpened(path) {
|
|
11
25
|
const db = new DatabaseSync(path, { readOnly: true });
|
|
12
26
|
const expected = new DatabaseSync(':memory:');
|
|
13
27
|
try {
|
|
14
28
|
db.exec('PRAGMA trusted_schema=OFF');
|
|
15
29
|
const version = bounded(db.prepare('PRAGMA user_version').get()?.user_version, Number.MAX_SAFE_INTEGER);
|
|
16
|
-
if (version >
|
|
17
|
-
throw new
|
|
30
|
+
if (version > migrations.length)
|
|
31
|
+
throw new SpongeError('schema-newer');
|
|
18
32
|
if (version < 1)
|
|
19
33
|
invalid();
|
|
20
34
|
for (let i = 0; i < version; i++) {
|
|
21
|
-
|
|
22
|
-
|
|
35
|
+
const { sql } = migrations[i];
|
|
36
|
+
expected.exec(sql);
|
|
37
|
+
if (db.prepare('SELECT checksum FROM schema_migrations WHERE version=?').get(i + 1)?.checksum !== sha256(sql))
|
|
23
38
|
invalid();
|
|
24
39
|
}
|
|
25
40
|
if (schema(db) !== schema(expected))
|
|
26
41
|
invalid();
|
|
27
42
|
if (db.prepare('SELECT count(*) n FROM schema_migrations').get()?.n !== version)
|
|
28
43
|
invalid();
|
|
29
|
-
if (db.prepare('PRAGMA quick_check').get()?.quick_check !== 'ok' ||
|
|
44
|
+
if (db.prepare('PRAGMA quick_check').get()?.quick_check !== 'ok' ||
|
|
45
|
+
db.prepare('PRAGMA foreign_key_check').all().length)
|
|
30
46
|
invalid();
|
|
31
47
|
const info = db.prepare('SELECT uuid,schema_version,revision_epoch FROM library_info WHERE id=1').get();
|
|
32
48
|
if (!info || typeof info.uuid !== 'string' || info.uuid.length > 256 || info.schema_version !== version)
|
|
33
49
|
invalid();
|
|
34
50
|
const count = Number(db.prepare('SELECT count(*) n FROM assets').get()?.n);
|
|
35
51
|
if (count > maximumEntries)
|
|
36
|
-
throw new
|
|
37
|
-
const blobs = db
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
52
|
+
throw new SpongeError('limit-exceeded');
|
|
53
|
+
const blobs = db
|
|
54
|
+
.prepare('SELECT sha256,bytes,relative_path FROM assets ORDER BY sha256')
|
|
55
|
+
.all()
|
|
56
|
+
.map((row) => {
|
|
57
|
+
const sha256 = digestString(row.sha256);
|
|
58
|
+
if (row.relative_path !== blobPath(sha256))
|
|
59
|
+
invalid();
|
|
60
|
+
return { sha256, bytes: bounded(row.bytes, maximumBlobBytes), relativePath: String(row.relative_path) };
|
|
61
|
+
});
|
|
62
|
+
for (const row of db
|
|
63
|
+
.prepare('SELECT markdown,sha256,bytes FROM representations WHERE markdown IS NOT NULL')
|
|
64
|
+
.iterate()) {
|
|
65
|
+
if (typeof row.markdown !== 'string' ||
|
|
66
|
+
sha256(row.markdown) !== row.sha256 ||
|
|
67
|
+
Buffer.byteLength(row.markdown) !== row.bytes)
|
|
41
68
|
invalid();
|
|
42
69
|
}
|
|
43
|
-
return {
|
|
70
|
+
return {
|
|
71
|
+
schemaVersion: version,
|
|
72
|
+
libraryId: info.uuid,
|
|
73
|
+
revisionEpoch: bounded(info.revision_epoch, Number.MAX_SAFE_INTEGER),
|
|
74
|
+
blobs,
|
|
75
|
+
};
|
|
44
76
|
}
|
|
45
77
|
finally {
|
|
46
78
|
expected.close();
|
|
@@ -55,9 +87,11 @@ export function neutralizeRestoredDatabase(path) {
|
|
|
55
87
|
try {
|
|
56
88
|
db.exec('PRAGMA trusted_schema=OFF; PRAGMA foreign_keys=ON; BEGIN IMMEDIATE');
|
|
57
89
|
// Every nonterminal intent is retained but requires an explicit retry after restore.
|
|
58
|
-
const held = db
|
|
90
|
+
const held = db
|
|
91
|
+
.prepare(`UPDATE jobs SET state='needs-permission',reason='restore-reconciliation-required'${inspected.schemaVersion >= 2 ? ',retryable=0' : ''} WHERE state NOT IN ('succeeded','failed','cancelled','unsupported')`)
|
|
92
|
+
.run();
|
|
59
93
|
db.prepare("UPDATE attempts SET stage='restored-unexecuted',cleanup='settled',finished_at=COALESCE(finished_at,?) WHERE cleanup='pending' OR finished_at IS NULL").run(new Date().toISOString());
|
|
60
|
-
db.exec(
|
|
94
|
+
db.exec('DELETE FROM staging_pins; DELETE FROM backup_fences; UPDATE worker_owner SET incarnation=NULL,fence=fence+1; UPDATE model_artifacts SET available=0;');
|
|
61
95
|
if (inspected.schemaVersion >= 4) {
|
|
62
96
|
db.exec("UPDATE index_jobs SET state='failed',reason='restore-reconciliation-required' WHERE state NOT IN ('succeeded','failed','cancelled')");
|
|
63
97
|
db.prepare("UPDATE index_attempts SET stage='restored-unexecuted',cleanup='settled',finished_at=COALESCE(finished_at,?) WHERE cleanup='pending' OR finished_at IS NULL").run(new Date().toISOString());
|
|
@@ -1,31 +1,57 @@
|
|
|
1
1
|
import { join } from 'node:path';
|
|
2
|
-
import { StoreError } from '../store/types.js';
|
|
3
2
|
import { safeDirectory } from '../store/files.js';
|
|
4
|
-
import { newDirectory, discardOwned, copyVerified,
|
|
3
|
+
import { newDirectory, discardOwned, copyVerified, blobPath, writePrivateExclusive } from './files.js';
|
|
4
|
+
import { syncDirectory } from '../util/fsync.js';
|
|
5
|
+
import { SpongeError } from '../errors.js';
|
|
5
6
|
/** Export a fixed revision with hash-named local assets. No network or acquisition. */
|
|
6
7
|
export function exportItem(store, itemId, output) {
|
|
7
8
|
const item = store.read(itemId);
|
|
8
9
|
if (!item)
|
|
9
|
-
throw new
|
|
10
|
+
throw new SpongeError('invalid-input');
|
|
10
11
|
const target = newDirectory(output);
|
|
11
12
|
try {
|
|
12
13
|
safeDirectory(join(output, 'assets'), true);
|
|
13
14
|
const known = new Map();
|
|
14
15
|
for (const asset of item.assets) {
|
|
15
16
|
if (asset.relativePath !== blobPath(asset.sha256))
|
|
16
|
-
throw new
|
|
17
|
+
throw new SpongeError('integrity');
|
|
17
18
|
const path = `assets/${asset.sha256}`;
|
|
18
19
|
if (!known.has(asset.sha256)) {
|
|
19
|
-
copyVerified(join(store.root, asset.relativePath), join(output, path), {
|
|
20
|
+
copyVerified(join(store.root, asset.relativePath), join(output, path), {
|
|
21
|
+
path,
|
|
22
|
+
bytes: asset.bytes,
|
|
23
|
+
sha256: asset.sha256,
|
|
24
|
+
});
|
|
20
25
|
known.set(asset.sha256, path);
|
|
21
26
|
}
|
|
22
27
|
}
|
|
23
|
-
const markdown = item.markdown.replace(/sponge-asset:([a-f0-9]{64})/gu, (_all, sha) => {
|
|
24
|
-
|
|
28
|
+
const markdown = item.markdown.replace(/sponge-asset:([a-f0-9]{64})/gu, (_all, sha) => {
|
|
29
|
+
const path = known.get(sha);
|
|
30
|
+
if (!path)
|
|
31
|
+
throw new SpongeError('integrity');
|
|
32
|
+
return path;
|
|
33
|
+
});
|
|
25
34
|
writePrivateExclusive(join(output, 'article.md'), markdown);
|
|
26
35
|
// Metadata can contain private descriptive content; exports stay private and explicit.
|
|
27
|
-
const notes = store.
|
|
28
|
-
writePrivateExclusive(join(output, 'item.json'), JSON.stringify({
|
|
36
|
+
const notes = store.annotations.forExport(itemId);
|
|
37
|
+
writePrivateExclusive(join(output, 'item.json'), JSON.stringify({
|
|
38
|
+
schemaVersion: 1,
|
|
39
|
+
...notes,
|
|
40
|
+
itemId: item.itemId,
|
|
41
|
+
revisionId: item.revisionId,
|
|
42
|
+
title: item.title,
|
|
43
|
+
kind: item.kind,
|
|
44
|
+
status: item.status,
|
|
45
|
+
metadata: item.metadata,
|
|
46
|
+
metadataWarnings: item.metadataWarnings,
|
|
47
|
+
assets: item.assets.map((a) => ({
|
|
48
|
+
sha256: a.sha256,
|
|
49
|
+
bytes: a.bytes,
|
|
50
|
+
mediaType: a.mediaType,
|
|
51
|
+
relationship: a.relationship,
|
|
52
|
+
path: known.get(a.sha256),
|
|
53
|
+
})),
|
|
54
|
+
}) + '\n');
|
|
29
55
|
syncDirectory(output);
|
|
30
56
|
return { output, itemId, revisionId: item.revisionId, assets: known.size };
|
|
31
57
|
}
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { syncDirectory } from '../store/files.js';
|
|
2
1
|
export declare const maximumTotalBytes: number, maximumDatabaseBytes: number, maximumBlobBytes: number, maximumEntries = 100000;
|
|
3
2
|
export interface FileRecord {
|
|
4
3
|
path: string;
|
|
@@ -6,7 +5,7 @@ export interface FileRecord {
|
|
|
6
5
|
sha256: string;
|
|
7
6
|
}
|
|
8
7
|
export declare function invalid(): never;
|
|
9
|
-
export declare
|
|
8
|
+
export declare const exact: (value: unknown, keys: string[]) => Record<string, unknown>;
|
|
10
9
|
export declare function bounded(value: unknown, max: number): number;
|
|
11
10
|
export declare function digestString(value: unknown): string;
|
|
12
11
|
export declare function blobPath(sha: string): string;
|
|
@@ -28,6 +27,4 @@ export declare function discardOwned(value: {
|
|
|
28
27
|
device: number;
|
|
29
28
|
inode: number;
|
|
30
29
|
}): void;
|
|
31
|
-
export declare function assertEmptyDirectory(path: string): void;
|
|
32
|
-
export { syncDirectory };
|
|
33
30
|
export declare function writePrivateExclusive(path: string, value: string): void;
|
|
@@ -1,78 +1,138 @@
|
|
|
1
1
|
import { createHash } from 'node:crypto';
|
|
2
|
-
import { constants, openSync, closeSync, fstatSync, readSync, writeSync, fsyncSync, lstatSync, mkdirSync,
|
|
2
|
+
import { constants, openSync, closeSync, fstatSync, readSync, writeSync, fsyncSync, lstatSync, mkdirSync, rmSync, } from 'node:fs';
|
|
3
3
|
import { dirname, resolve } from 'node:path';
|
|
4
|
-
import { safeDirectory, regularFile
|
|
5
|
-
import {
|
|
4
|
+
import { safeDirectory, regularFile } from '../store/files.js';
|
|
5
|
+
import { syncDirectory } from '../util/fsync.js';
|
|
6
|
+
import { exactKeys } from '../util/exact.js';
|
|
7
|
+
import { SpongeError } from '../errors.js';
|
|
6
8
|
export const maximumTotalBytes = 16 * 1024 ** 3, maximumDatabaseBytes = 512 * 1024 ** 2, maximumBlobBytes = 512 * 1024 ** 2, maximumEntries = 100_000;
|
|
7
|
-
export function invalid() {
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
export
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
invalid(); return value; }
|
|
14
|
-
export function blobPath(sha) { return `blobs/sha256/${sha.slice(0, 2)}/${sha}`; }
|
|
15
|
-
export function fileDescriptor(value) { const r = exact(value, ['path', 'bytes', 'sha256']); const sha256 = digestString(r.sha256); if (r.path !== 'library.sqlite' && r.path !== blobPath(sha256))
|
|
16
|
-
invalid(); return { path: String(r.path), bytes: bounded(r.bytes, r.path === 'library.sqlite' ? maximumDatabaseBytes : maximumBlobBytes), sha256 }; }
|
|
17
|
-
export function openRead(path) { safeDirectory(dirname(path)); regularFile(path); const fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW); const stat = fstatSync(fd); if (!stat.isFile() || stat.nlink !== 1) {
|
|
18
|
-
closeSync(fd);
|
|
19
|
-
invalid();
|
|
20
|
-
} return fd; }
|
|
21
|
-
export function writeAll(fd, bytes) { let n = 0; while (n < bytes.length) {
|
|
22
|
-
const written = writeSync(fd, bytes, n, bytes.length - n);
|
|
23
|
-
if (!written)
|
|
9
|
+
export function invalid() {
|
|
10
|
+
throw new SpongeError('integrity');
|
|
11
|
+
}
|
|
12
|
+
export const exact = (value, keys) => exactKeys(value, keys, invalid);
|
|
13
|
+
export function bounded(value, max) {
|
|
14
|
+
if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 0 || value > max)
|
|
24
15
|
invalid();
|
|
25
|
-
|
|
26
|
-
}
|
|
27
|
-
export function
|
|
28
|
-
|
|
29
|
-
if (!read)
|
|
16
|
+
return value;
|
|
17
|
+
}
|
|
18
|
+
export function digestString(value) {
|
|
19
|
+
if (typeof value !== 'string' || !/^[a-f0-9]{64}$/u.test(value))
|
|
30
20
|
invalid();
|
|
31
|
-
|
|
32
|
-
}
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
21
|
+
return value;
|
|
22
|
+
}
|
|
23
|
+
export function blobPath(sha) {
|
|
24
|
+
return `blobs/sha256/${sha.slice(0, 2)}/${sha}`;
|
|
25
|
+
}
|
|
26
|
+
export function fileDescriptor(value) {
|
|
27
|
+
const r = exact(value, ['path', 'bytes', 'sha256']);
|
|
28
|
+
const sha256 = digestString(r.sha256);
|
|
29
|
+
if (r.path !== 'library.sqlite' && r.path !== blobPath(sha256))
|
|
37
30
|
invalid();
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
}
|
|
43
|
-
export function describe(path, relativePath, maximum) { const fd = openRead(path); try {
|
|
44
|
-
const bytes = bounded(fstatSync(fd).size, maximum);
|
|
45
|
-
return { path: relativePath, bytes, sha256: transfer(fd, bytes) };
|
|
31
|
+
return {
|
|
32
|
+
path: String(r.path),
|
|
33
|
+
bytes: bounded(r.bytes, r.path === 'library.sqlite' ? maximumDatabaseBytes : maximumBlobBytes),
|
|
34
|
+
sha256,
|
|
35
|
+
};
|
|
46
36
|
}
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
37
|
+
export function openRead(path) {
|
|
38
|
+
safeDirectory(dirname(path));
|
|
39
|
+
regularFile(path);
|
|
40
|
+
const fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW);
|
|
41
|
+
const stat = fstatSync(fd);
|
|
42
|
+
if (!stat.isFile() || stat.nlink !== 1) {
|
|
43
|
+
closeSync(fd);
|
|
52
44
|
invalid();
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
45
|
+
}
|
|
46
|
+
return fd;
|
|
47
|
+
}
|
|
48
|
+
export function writeAll(fd, bytes) {
|
|
49
|
+
let n = 0;
|
|
50
|
+
while (n < bytes.length) {
|
|
51
|
+
const written = writeSync(fd, bytes, n, bytes.length - n);
|
|
52
|
+
if (!written)
|
|
53
|
+
invalid();
|
|
54
|
+
n += written;
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
export function readExact(fd, bytes) {
|
|
58
|
+
const result = Buffer.alloc(bytes);
|
|
59
|
+
let n = 0;
|
|
60
|
+
while (n < bytes) {
|
|
61
|
+
const read = readSync(fd, result, n, bytes - n, null);
|
|
62
|
+
if (!read)
|
|
63
|
+
invalid();
|
|
64
|
+
n += read;
|
|
65
|
+
}
|
|
66
|
+
return result;
|
|
67
|
+
}
|
|
68
|
+
/** Stream only expected bytes from one descriptor; never allocate a full PDF or archive. */
|
|
69
|
+
export function transfer(input, bytes, output) {
|
|
70
|
+
const hash = createHash('sha256'), buffer = Buffer.alloc(64 * 1024);
|
|
71
|
+
let n = 0;
|
|
72
|
+
while (n < bytes) {
|
|
73
|
+
const read = readSync(input, buffer, 0, Math.min(buffer.length, bytes - n), null);
|
|
74
|
+
if (!read)
|
|
75
|
+
invalid();
|
|
76
|
+
hash.update(buffer.subarray(0, read));
|
|
77
|
+
if (output !== undefined)
|
|
78
|
+
writeAll(output, buffer.subarray(0, read));
|
|
79
|
+
n += read;
|
|
80
|
+
}
|
|
81
|
+
return hash.digest('hex');
|
|
82
|
+
}
|
|
83
|
+
export function describe(path, relativePath, maximum) {
|
|
84
|
+
const fd = openRead(path);
|
|
85
|
+
try {
|
|
86
|
+
const bytes = bounded(fstatSync(fd).size, maximum);
|
|
87
|
+
return { path: relativePath, bytes, sha256: transfer(fd, bytes) };
|
|
88
|
+
}
|
|
89
|
+
finally {
|
|
90
|
+
closeSync(fd);
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
export function copyVerified(source, target, record) {
|
|
94
|
+
const input = openRead(source);
|
|
95
|
+
let output;
|
|
96
|
+
try {
|
|
97
|
+
if (fstatSync(input).size !== record.bytes)
|
|
98
|
+
invalid();
|
|
99
|
+
safeDirectory(dirname(target), true);
|
|
100
|
+
output = openSync(target, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600);
|
|
101
|
+
if (transfer(input, record.bytes, output) !== record.sha256)
|
|
102
|
+
invalid();
|
|
103
|
+
fsyncSync(output);
|
|
104
|
+
syncDirectory(dirname(target));
|
|
105
|
+
}
|
|
106
|
+
finally {
|
|
107
|
+
closeSync(input);
|
|
108
|
+
if (output !== undefined)
|
|
109
|
+
closeSync(output);
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
export function newDirectory(path) {
|
|
113
|
+
if (resolve(path) !== path)
|
|
114
|
+
throw new SpongeError('invalid-input');
|
|
115
|
+
safeDirectory(dirname(path));
|
|
116
|
+
mkdirSync(path, { mode: 0o700 });
|
|
117
|
+
const stat = lstatSync(path);
|
|
118
|
+
return { root: path, device: stat.dev, inode: stat.ino };
|
|
119
|
+
}
|
|
120
|
+
export function discardOwned(value) {
|
|
121
|
+
const stat = lstatSync(value.root);
|
|
122
|
+
if (!stat.isDirectory() || stat.isSymbolicLink() || stat.dev !== value.device || stat.ino !== value.inode)
|
|
56
123
|
invalid();
|
|
57
|
-
|
|
58
|
-
syncDirectory(dirname(
|
|
124
|
+
rmSync(value.root, { recursive: true });
|
|
125
|
+
syncDirectory(dirname(value.root));
|
|
59
126
|
}
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
export { syncDirectory };
|
|
72
|
-
export function writePrivateExclusive(path, value) { safeDirectory(dirname(path)); const fd = openSync(path, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600); try {
|
|
73
|
-
writeAll(fd, Buffer.from(value));
|
|
74
|
-
fsyncSync(fd);
|
|
127
|
+
export function writePrivateExclusive(path, value) {
|
|
128
|
+
safeDirectory(dirname(path));
|
|
129
|
+
const fd = openSync(path, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600);
|
|
130
|
+
try {
|
|
131
|
+
writeAll(fd, Buffer.from(value));
|
|
132
|
+
fsyncSync(fd);
|
|
133
|
+
}
|
|
134
|
+
finally {
|
|
135
|
+
closeSync(fd);
|
|
136
|
+
}
|
|
137
|
+
syncDirectory(dirname(path));
|
|
75
138
|
}
|
|
76
|
-
finally {
|
|
77
|
-
closeSync(fd);
|
|
78
|
-
} syncDirectory(dirname(path)); }
|