@dreamlake/dreamlake-cli 0.101.0 → 0.102.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/host-moves.d.ts +20 -0
- package/dist/api/host-moves.js +18 -0
- package/dist/api/index.d.ts +1 -1
- package/dist/cli/artifact/incremental.d.ts +6 -1
- package/dist/cli/artifact/incremental.js +69 -7
- package/dist/cli/artifact/index.js +13 -3
- package/dist/cli/auth/materialized.d.ts +6 -0
- package/dist/cli/auth/materialized.js +2 -1
- package/dist/cli/compat/dreamdb.d.ts +11 -0
- package/dist/cli/compat/dreamdb.js +57 -0
- package/dist/cli/compat/s3-backend.d.ts +9 -0
- package/dist/cli/compat/s3-backend.js +164 -0
- package/dist/cli/config.d.ts +2 -2
- package/dist/cli/config.js +13 -5
- package/dist/cli/help-examples.generated.js +1 -0
- package/dist/cli/hosts/index.js +5 -5
- package/dist/cli/hosts/move.d.ts +3 -0
- package/dist/cli/hosts/move.js +5 -0
- package/dist/cli/hosts/ssh-transport.d.ts +27 -0
- package/dist/cli/hosts/ssh-transport.js +79 -0
- package/dist/cli/hosts/ssh.js +2 -3
- package/dist/cli/index.js +2 -0
- package/dist/cli/infra/index.js +2 -2
- package/dist/cli/jobs/index.js +1 -1
- package/dist/cli/namespace.d.ts +10 -0
- package/dist/cli/namespace.js +70 -0
- package/dist/cli/notes/index.js +31 -33
- package/dist/cli/version.d.ts +1 -1
- package/dist/cli/version.js +1 -1
- package/package.json +10 -9
package/dist/api/host-moves.d.ts
CHANGED
|
@@ -1,4 +1,20 @@
|
|
|
1
1
|
import { type Transport } from './client.js';
|
|
2
|
+
/** Saved sessions stay at their original addresses; existing sharing is removed. */
|
|
3
|
+
export type HostMoveRetainedSessions = {
|
|
4
|
+
count: number;
|
|
5
|
+
policy: 'retain_original_namespace_make_private';
|
|
6
|
+
accessChanges: {
|
|
7
|
+
public: number;
|
|
8
|
+
shared: number;
|
|
9
|
+
};
|
|
10
|
+
};
|
|
11
|
+
/** Run history and acknowledged executions retain their original namespace and identities. */
|
|
12
|
+
export type HostMoveRetainedRuns = {
|
|
13
|
+
count: number;
|
|
14
|
+
terminal: number;
|
|
15
|
+
active: number;
|
|
16
|
+
policy: 'retain_original_namespace_acknowledged_runs';
|
|
17
|
+
};
|
|
2
18
|
export type HostMovePreview = {
|
|
3
19
|
apiVersion: 1;
|
|
4
20
|
hostId: string;
|
|
@@ -12,6 +28,8 @@ export type HostMovePreview = {
|
|
|
12
28
|
previewHash: string;
|
|
13
29
|
blockers: string[];
|
|
14
30
|
executable: boolean;
|
|
31
|
+
sessions?: HostMoveRetainedSessions;
|
|
32
|
+
runs?: HostMoveRetainedRuns;
|
|
15
33
|
};
|
|
16
34
|
export type HostMoveRequest = {
|
|
17
35
|
destinationNamespace: string;
|
|
@@ -25,6 +43,8 @@ export type HostMoveReceipt = Omit<HostMovePreview, 'previewHash' | 'blockers' |
|
|
|
25
43
|
recordedAt: string;
|
|
26
44
|
replayed: boolean;
|
|
27
45
|
};
|
|
46
|
+
export declare function validateHostMoveSessions(value: unknown): void;
|
|
47
|
+
export declare function validateHostMoveRuns(value: unknown): void;
|
|
28
48
|
export declare function validateHostMovePreview(value: unknown, hostId: string, destinationNamespace: string): HostMovePreview;
|
|
29
49
|
export declare function validateHostMoveReceipt(value: unknown, hostId: string, request: HostMoveRequest): HostMoveReceipt;
|
|
30
50
|
export declare function hostMoveApi(t: Transport, namespace: string): {
|
package/dist/api/host-moves.js
CHANGED
|
@@ -6,11 +6,29 @@ const base = (namespace, hostId) => {
|
|
|
6
6
|
invalid();
|
|
7
7
|
return `/namespaces/${seg(namespace)}/hosts/${hostId}/move`;
|
|
8
8
|
};
|
|
9
|
+
export function validateHostMoveSessions(value) {
|
|
10
|
+
if (value === undefined)
|
|
11
|
+
return;
|
|
12
|
+
const v = value;
|
|
13
|
+
if (!v || v.policy !== 'retain_original_namespace_make_private' || !Number.isSafeInteger(v.count) || v.count < 0 ||
|
|
14
|
+
!v.accessChanges || !['public', 'shared'].every(key => { const n = v.accessChanges[key]; return Number.isSafeInteger(n) && n >= 0 && n <= v.count; }))
|
|
15
|
+
invalid();
|
|
16
|
+
}
|
|
17
|
+
export function validateHostMoveRuns(value) {
|
|
18
|
+
if (value === undefined)
|
|
19
|
+
return;
|
|
20
|
+
const v = value;
|
|
21
|
+
if (!v || v.policy !== 'retain_original_namespace_acknowledged_runs' ||
|
|
22
|
+
![v.count, v.terminal, v.active].every(n => Number.isSafeInteger(n) && n >= 0) || v.terminal + v.active !== v.count)
|
|
23
|
+
invalid();
|
|
24
|
+
}
|
|
9
25
|
function common(v, hostId, destinationNamespace) {
|
|
10
26
|
if (!v || v.apiVersion !== 1 || v.hostId !== hostId || v.destinationNamespace !== destinationNamespace ||
|
|
11
27
|
![v.sourceNamespaceId, v.destinationNamespaceId, v.ownerUserId, v.name].every(nonempty) ||
|
|
12
28
|
v.visibility !== 'private' || !Number.isSafeInteger(v.revision) || v.revision < 0)
|
|
13
29
|
invalid();
|
|
30
|
+
validateHostMoveSessions(v.sessions);
|
|
31
|
+
validateHostMoveRuns(v.runs);
|
|
14
32
|
}
|
|
15
33
|
export function validateHostMovePreview(value, hostId, destinationNamespace) {
|
|
16
34
|
const v = value;
|
package/dist/api/index.d.ts
CHANGED
|
@@ -2122,4 +2122,4 @@ export { documentEdits, documentInsertion } from "./note-edits.js";
|
|
|
2122
2122
|
export * from "./note-facade.js";
|
|
2123
2123
|
export type { NoteReadingOptions, NoteReadingPage, NoteFocusedRead } from "./note-reading.js";
|
|
2124
2124
|
export { lineRange, focusedRead, InvalidLineRange } from "./note-reading.js";
|
|
2125
|
-
export type { HostMovePreview, HostMoveRequest, HostMoveReceipt } from "./host-moves.js";
|
|
2125
|
+
export type { HostMovePreview, HostMoveRequest, HostMoveReceipt, HostMoveRetainedSessions, HostMoveRetainedRuns } from "./host-moves.js";
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import type { DreamdbBackend } from "../compat/s3-backend.js";
|
|
2
2
|
export declare const MAGIC = "DREAMLAKE-ARTIFACT-CHUNKS/1\n";
|
|
3
3
|
export declare const MAX_SIZE: number;
|
|
4
|
+
export declare const MAX_MANIFEST_SIZE: number;
|
|
5
|
+
export declare const MAX_CHUNK: number;
|
|
4
6
|
export interface Chunk {
|
|
5
7
|
sha256: string;
|
|
6
8
|
size: number;
|
|
@@ -10,12 +12,15 @@ export interface Manifest {
|
|
|
10
12
|
sha256: string;
|
|
11
13
|
chunks: Chunk[];
|
|
12
14
|
}
|
|
15
|
+
/** An optional optimization hint, never proof that its objects still exist. */
|
|
16
|
+
export declare function parseManifest(bytes?: Uint8Array | null): Manifest | null;
|
|
13
17
|
/** Content-defined boundaries resynchronize after insertions/deletions. */
|
|
14
18
|
export declare function splitChunks(bytes: Uint8Array): Uint8Array[];
|
|
15
19
|
/** Chunks finish before the caller publishes the manifest through DreamDB CAS. */
|
|
16
|
-
export declare function uploadIncremental(content: Uint8Array, backend: DreamdbBackend): Promise<{
|
|
20
|
+
export declare function uploadIncremental(content: Uint8Array, backend: DreamdbBackend, previousContent?: Uint8Array | null): Promise<{
|
|
17
21
|
content: NodeJS.NonSharedUint8Array;
|
|
18
22
|
uploadedBytes: number;
|
|
19
23
|
uploadedChunks: number;
|
|
20
24
|
reusedBytes: number;
|
|
25
|
+
reusedManifest: boolean;
|
|
21
26
|
}>;
|
|
@@ -4,8 +4,9 @@ import { promisify } from "node:util";
|
|
|
4
4
|
// Wire contract: UI docs/dev-notes/2026-09-30-artifact-incremental.md.
|
|
5
5
|
export const MAGIC = "DREAMLAKE-ARTIFACT-CHUNKS/1\n";
|
|
6
6
|
export const MAX_SIZE = 512 * 1024 * 1024;
|
|
7
|
+
export const MAX_MANIFEST_SIZE = 1024 * 1024;
|
|
7
8
|
const MIN_CHUNK = 64 * 1024;
|
|
8
|
-
const MAX_CHUNK = 1024 * 1024;
|
|
9
|
+
export const MAX_CHUNK = 1024 * 1024;
|
|
9
10
|
const compress = promisify(gzip);
|
|
10
11
|
const sha256 = (bytes) => createHash("sha256").update(bytes).digest("hex");
|
|
11
12
|
let seed = 0x12345678;
|
|
@@ -15,6 +16,55 @@ const gear = Uint32Array.from({ length: 256 }, () => {
|
|
|
15
16
|
seed ^= seed << 5;
|
|
16
17
|
return seed >>> 0;
|
|
17
18
|
});
|
|
19
|
+
/** An optional optimization hint, never proof that its objects still exist. */
|
|
20
|
+
export function parseManifest(bytes) {
|
|
21
|
+
if (!bytes || bytes.length > MAX_MANIFEST_SIZE)
|
|
22
|
+
return null;
|
|
23
|
+
const text = new TextDecoder().decode(bytes);
|
|
24
|
+
if (!text.startsWith(MAGIC))
|
|
25
|
+
return null;
|
|
26
|
+
try {
|
|
27
|
+
const value = JSON.parse(text.slice(MAGIC.length));
|
|
28
|
+
const hash = (v) => typeof v === 'string' && /^[a-f0-9]{64}$/.test(v);
|
|
29
|
+
if (!value || !Number.isSafeInteger(value.size) || value.size < 0 || value.size > MAX_SIZE ||
|
|
30
|
+
!hash(value.sha256) || !Array.isArray(value.chunks) || value.chunks.length > 8192)
|
|
31
|
+
return null;
|
|
32
|
+
let total = 0;
|
|
33
|
+
const sizes = new Map();
|
|
34
|
+
const chunks = [];
|
|
35
|
+
for (const chunk of value.chunks) {
|
|
36
|
+
if (!chunk || !hash(chunk.sha256) || !Number.isSafeInteger(chunk.size) ||
|
|
37
|
+
chunk.size <= 0 || chunk.size > MAX_CHUNK ||
|
|
38
|
+
(sizes.has(chunk.sha256) && sizes.get(chunk.sha256) !== chunk.size))
|
|
39
|
+
return null;
|
|
40
|
+
total += chunk.size;
|
|
41
|
+
if (total > value.size)
|
|
42
|
+
return null;
|
|
43
|
+
sizes.set(chunk.sha256, chunk.size);
|
|
44
|
+
chunks.push({ sha256: chunk.sha256, size: chunk.size });
|
|
45
|
+
}
|
|
46
|
+
return total === value.size ? { size: value.size, sha256: value.sha256, chunks } : null;
|
|
47
|
+
}
|
|
48
|
+
catch {
|
|
49
|
+
return null;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
/** Verify the prior boundaries against the local bytes, without CDC scanning.
|
|
53
|
+
* A forged/corrupt envelope with a matching whole-file hash must not be reused. */
|
|
54
|
+
function unchangedParts(content, hash, previous) {
|
|
55
|
+
if (!previous || previous.size !== content.length || previous.sha256 !== hash)
|
|
56
|
+
return null;
|
|
57
|
+
const parts = [];
|
|
58
|
+
let offset = 0;
|
|
59
|
+
for (const chunk of previous.chunks) {
|
|
60
|
+
const part = content.subarray(offset, offset + chunk.size);
|
|
61
|
+
if (sha256(part) !== chunk.sha256)
|
|
62
|
+
return null;
|
|
63
|
+
parts.push(part);
|
|
64
|
+
offset += chunk.size;
|
|
65
|
+
}
|
|
66
|
+
return parts;
|
|
67
|
+
}
|
|
18
68
|
/** Content-defined boundaries resynchronize after insertions/deletions. */
|
|
19
69
|
export function splitChunks(bytes) {
|
|
20
70
|
if (bytes.length > MAX_SIZE)
|
|
@@ -35,16 +85,27 @@ export function splitChunks(bytes) {
|
|
|
35
85
|
return result;
|
|
36
86
|
}
|
|
37
87
|
/** Chunks finish before the caller publishes the manifest through DreamDB CAS. */
|
|
38
|
-
export async function uploadIncremental(content, backend) {
|
|
39
|
-
|
|
40
|
-
|
|
88
|
+
export async function uploadIncremental(content, backend, previousContent) {
|
|
89
|
+
if (content.length > MAX_SIZE)
|
|
90
|
+
throw new Error("incremental artifacts are limited to 512 MiB");
|
|
91
|
+
const contentHash = sha256(content);
|
|
92
|
+
const previous = parseManifest(previousContent);
|
|
93
|
+
const unchanged = unchangedParts(content, contentHash, previous);
|
|
94
|
+
const parts = unchanged ?? splitChunks(content);
|
|
95
|
+
const manifest = { size: content.length, sha256: contentHash, chunks: [] };
|
|
41
96
|
const unique = new Map();
|
|
42
|
-
for (const part of parts) {
|
|
43
|
-
const hash = sha256(part);
|
|
97
|
+
for (const [index, part] of parts.entries()) {
|
|
98
|
+
const hash = unchanged ? previous.chunks[index].sha256 : sha256(part);
|
|
44
99
|
manifest.chunks.push({ sha256: hash, size: part.length });
|
|
45
100
|
unique.set(hash, part);
|
|
46
101
|
}
|
|
47
102
|
const entries = [...unique];
|
|
103
|
+
// One bounded batched storage query when supported. Older/custom backends
|
|
104
|
+
// retain their per-object HEAD path. A prior manifest is not existence proof:
|
|
105
|
+
// a missing object is repaired from the local bytes before publication.
|
|
106
|
+
const existing = backend.hasMany && entries.length
|
|
107
|
+
? await backend.hasMany(entries.map(([hash]) => `artifact-chunks/v1/${hash}.gz`))
|
|
108
|
+
: null;
|
|
48
109
|
let next = 0, uploadedBytes = 0, uploadedSourceBytes = 0, uploadedChunks = 0;
|
|
49
110
|
let failure;
|
|
50
111
|
const workers = Array.from({ length: Math.min(8, entries.length) }, async () => {
|
|
@@ -52,7 +113,7 @@ export async function uploadIncremental(content, backend) {
|
|
|
52
113
|
const [hash, part] = entries[next++];
|
|
53
114
|
try {
|
|
54
115
|
const key = `artifact-chunks/v1/${hash}.gz`;
|
|
55
|
-
if ((await backend.head(key)).exists)
|
|
116
|
+
if (existing ? existing.has(key) : (await backend.head(key)).exists)
|
|
56
117
|
continue;
|
|
57
118
|
const compressed = await compress(part);
|
|
58
119
|
const result = await backend.put(key, compressed, { ifNoneMatchStar: true });
|
|
@@ -76,5 +137,6 @@ export async function uploadIncremental(content, backend) {
|
|
|
76
137
|
content: new TextEncoder().encode(MAGIC + JSON.stringify(manifest)),
|
|
77
138
|
uploadedBytes, uploadedChunks,
|
|
78
139
|
reusedBytes: content.length - uploadedSourceBytes,
|
|
140
|
+
reusedManifest: unchanged !== null,
|
|
79
141
|
};
|
|
80
142
|
}
|
|
@@ -9,7 +9,7 @@ import { registerShareCommands } from "../share.js";
|
|
|
9
9
|
// never transits the API server. See compat/dreamdb.ts.
|
|
10
10
|
import { readFileSync, statSync } from "node:fs";
|
|
11
11
|
import path from "node:path";
|
|
12
|
-
import { uploadIncremental, MAX_SIZE } from "./incremental.js";
|
|
12
|
+
import { uploadIncremental, MAX_SIZE, MAX_MANIFEST_SIZE } from "./incremental.js";
|
|
13
13
|
import { packPages } from "./pages.js";
|
|
14
14
|
import { collectSite, uploadSite } from "./site.js";
|
|
15
15
|
import { metaError, putArtifactCover, readCoverFile, registerArtifactMetaCommand } from "./meta.js";
|
|
@@ -197,8 +197,18 @@ export async function runDreamlakeArtifactPush(args, deps = {}) {
|
|
|
197
197
|
const io = deps.dreamdb ?? realDreamdbIO;
|
|
198
198
|
// 2) write the version straight to S3 via DreamDB.
|
|
199
199
|
let version;
|
|
200
|
+
let previousContent = null;
|
|
200
201
|
try {
|
|
201
|
-
|
|
202
|
+
// Avoid extra index/blob reads on small/medium inputs, where CDC is cheap.
|
|
203
|
+
// This is only an optimization threshold, not an upload size limit.
|
|
204
|
+
if (incremental && !site && content.length >= 64 * 1024 * 1024 && io.readHead) {
|
|
205
|
+
const head = await io.readHead(broker.backendUrl, refName, backend, "artifact_id", artifactId, MAX_MANIFEST_SIZE);
|
|
206
|
+
version = head.nextVersion;
|
|
207
|
+
previousContent = head.content;
|
|
208
|
+
}
|
|
209
|
+
else {
|
|
210
|
+
version = await io.nextVersion(broker.backendUrl, refName, backend, "artifact_id", artifactId);
|
|
211
|
+
}
|
|
202
212
|
}
|
|
203
213
|
catch (e) {
|
|
204
214
|
// Abort BEFORE writing — never guess v1 on a read failure and clobber
|
|
@@ -217,7 +227,7 @@ export async function runDreamlakeArtifactPush(args, deps = {}) {
|
|
|
217
227
|
process.stdout.write(` site: ${site.manifest.files.length} files, ${result.uploadedFiles} new files, ${result.uploadedBytes} bytes uploaded\n`);
|
|
218
228
|
}
|
|
219
229
|
else if (incremental) {
|
|
220
|
-
const result = await uploadIncremental(content, backend);
|
|
230
|
+
const result = await uploadIncremental(content, backend, previousContent);
|
|
221
231
|
storedContent = result.content;
|
|
222
232
|
process.stdout.write(` incremental: ${result.uploadedChunks} new chunks, ${result.uploadedBytes} compressed bytes uploaded, ${result.reusedBytes} source bytes reused\n`);
|
|
223
233
|
}
|
|
@@ -10,6 +10,12 @@ export interface MaterializedConfig {
|
|
|
10
10
|
remote_url?: string;
|
|
11
11
|
/** Cached namespace slug for the materialized (remote, token) pair. */
|
|
12
12
|
namespace?: string;
|
|
13
|
+
/** Local CLI default, bound to the validated connection without storing another bearer. */
|
|
14
|
+
default_namespace?: {
|
|
15
|
+
namespace: string;
|
|
16
|
+
remote: string;
|
|
17
|
+
tokenSha256: string;
|
|
18
|
+
};
|
|
13
19
|
device_secret?: string;
|
|
14
20
|
[key: string]: unknown;
|
|
15
21
|
}
|
|
@@ -91,6 +91,7 @@ export function projectEnv(env, dir) {
|
|
|
91
91
|
updateConfig({
|
|
92
92
|
remote_url: env.server.replace(/\/+$/, ""),
|
|
93
93
|
namespace: env.namespace || undefined,
|
|
94
|
+
default_namespace: undefined,
|
|
94
95
|
}, dir);
|
|
95
96
|
}
|
|
96
97
|
/** Remove the connection (logout with no env left). Keeps device_secret. */
|
|
@@ -98,5 +99,5 @@ export function clearProjection(dir) {
|
|
|
98
99
|
const tokens = readJson(tokensPath(dir));
|
|
99
100
|
delete tokens[TOKEN_SLOT];
|
|
100
101
|
writeJson(tokensPath(dir), tokens);
|
|
101
|
-
updateConfig({ remote_url: undefined, namespace: undefined }, dir);
|
|
102
|
+
updateConfig({ remote_url: undefined, namespace: undefined, default_namespace: undefined }, dir);
|
|
102
103
|
}
|
|
@@ -57,6 +57,16 @@ export declare function backendFrom(broker: UploadCredentials): S3Backend;
|
|
|
57
57
|
* says so.
|
|
58
58
|
*/
|
|
59
59
|
export declare function nextVersion(backendUrl: string, refName: string, backend: DreamdbBackend, idField: string, idValue: string): Promise<number>;
|
|
60
|
+
export interface DreamdbHead {
|
|
61
|
+
nextVersion: number;
|
|
62
|
+
/** Optional bounded content from the newest matching row, in this snapshot. */
|
|
63
|
+
content: Uint8Array | null;
|
|
64
|
+
}
|
|
65
|
+
/** Read version and an optional small prior envelope from the same current
|
|
66
|
+
* broker-selected dataset snapshot. Never fetch a large historical raw blob.
|
|
67
|
+
* Ref/version read failures propagate; unavailable prior content merely
|
|
68
|
+
* disables the optimization, without guessing a new version number. */
|
|
69
|
+
export declare function readHead(backendUrl: string, refName: string, backend: DreamdbBackend, idField: string, idValue: string, maxContentBytes: number): Promise<DreamdbHead>;
|
|
60
70
|
/**
|
|
61
71
|
* A writer for the dataset, creating it on first push.
|
|
62
72
|
*
|
|
@@ -68,6 +78,7 @@ export interface DreamdbRow {
|
|
|
68
78
|
[field: string]: unknown;
|
|
69
79
|
}
|
|
70
80
|
export interface DreamdbIO {
|
|
81
|
+
readHead?(backendUrl: string, refName: string, backend: DreamdbBackend, idField: string, idValue: string, maxContentBytes: number): Promise<DreamdbHead>;
|
|
71
82
|
nextVersion(backendUrl: string, refName: string, backend: DreamdbBackend, idField: string, idValue: string): Promise<number>;
|
|
72
83
|
writeRow(backendUrl: string, refName: string, schema: SchemaField[], backend: DreamdbBackend, row: DreamdbRow): Promise<void>;
|
|
73
84
|
}
|
|
@@ -106,6 +106,62 @@ export async function nextVersion(backendUrl, refName, backend, idField, idValue
|
|
|
106
106
|
}
|
|
107
107
|
return max + 1;
|
|
108
108
|
}
|
|
109
|
+
/** Read version and an optional small prior envelope from the same current
|
|
110
|
+
* broker-selected dataset snapshot. Never fetch a large historical raw blob.
|
|
111
|
+
* Ref/version read failures propagate; unavailable prior content merely
|
|
112
|
+
* disables the optimization, without guessing a new version number. */
|
|
113
|
+
export async function readHead(backendUrl, refName, backend, idField, idValue, maxContentBytes) {
|
|
114
|
+
if (!Number.isSafeInteger(maxContentBytes) || maxContentBytes < 0)
|
|
115
|
+
throw new Error('Invalid prior content limit');
|
|
116
|
+
if (!(await datasetExists(backendUrl, refName, backend)))
|
|
117
|
+
return { nextVersion: 1, content: null };
|
|
118
|
+
const space = await Space.fromUri(refUri(backendUrl, refName), backend);
|
|
119
|
+
const anchors = (await space.queryScalar(idField, 'eq', idValue));
|
|
120
|
+
const wanted = new Set(anchors.map(String));
|
|
121
|
+
if (!wanted.size)
|
|
122
|
+
return { nextVersion: 1, content: null };
|
|
123
|
+
const column = await space.readScalarColumn(space.trackFor('version'), {});
|
|
124
|
+
let max = 0;
|
|
125
|
+
let anchor = null;
|
|
126
|
+
for (const [key, value] of column) {
|
|
127
|
+
if (!wanted.has(String(key)))
|
|
128
|
+
continue;
|
|
129
|
+
const version = Number(value);
|
|
130
|
+
if (!Number.isFinite(version))
|
|
131
|
+
continue;
|
|
132
|
+
if (version > max) {
|
|
133
|
+
max = version;
|
|
134
|
+
anchor = String(key);
|
|
135
|
+
}
|
|
136
|
+
else if (version === max)
|
|
137
|
+
anchor = null; // Ambiguous history: do not reuse.
|
|
138
|
+
}
|
|
139
|
+
const head = { nextVersion: max + 1, content: null };
|
|
140
|
+
if (!anchor)
|
|
141
|
+
return head;
|
|
142
|
+
try {
|
|
143
|
+
const track = space.tracks()
|
|
144
|
+
.find(t => String(t.modality).startsWith('image'));
|
|
145
|
+
if (!track)
|
|
146
|
+
return head;
|
|
147
|
+
const index = await space.resolveObjectIndex(track);
|
|
148
|
+
const hits = index.filter(e => e.length >= 4 && String(e[0]) === anchor && e[3] instanceof Uint8Array);
|
|
149
|
+
if (hits.length !== 1)
|
|
150
|
+
return head;
|
|
151
|
+
const hit = hits[0];
|
|
152
|
+
const size = Number(hit[2]);
|
|
153
|
+
if (!Number.isSafeInteger(size) || size < 0 || size > maxContentBytes)
|
|
154
|
+
return head;
|
|
155
|
+
const { bytes } = await backend.get(`${track.timeline}/${track.modality}/${ZERO_SPATIAL_KEY}/${bytesToBase32(hit[3])}`, { start: 0, end: maxContentBytes + 1 });
|
|
156
|
+
if (bytes.length <= maxContentBytes && bytes.length === size)
|
|
157
|
+
head.content = bytes;
|
|
158
|
+
}
|
|
159
|
+
catch {
|
|
160
|
+
// Missing/corrupt/unsupported prior content is repaired by a normal push.
|
|
161
|
+
// The version above came from a successful authoritative read, not a cache.
|
|
162
|
+
}
|
|
163
|
+
return head;
|
|
164
|
+
}
|
|
109
165
|
/** Whether `refs/<refName>` is published — i.e. the dataset has been created. */
|
|
110
166
|
async function datasetExists(backendUrl, refName, backend) {
|
|
111
167
|
void backendUrl; // the ref key is relative to the dataset base
|
|
@@ -126,6 +182,7 @@ export async function openOrCreateWriter(backendUrl, refName, schema, backend) {
|
|
|
126
182
|
return authoring.writer();
|
|
127
183
|
}
|
|
128
184
|
export const realDreamdbIO = {
|
|
185
|
+
readHead,
|
|
129
186
|
nextVersion,
|
|
130
187
|
async writeRow(backendUrl, refName, schema, backend, row) {
|
|
131
188
|
const writer = await openOrCreateWriter(backendUrl, refName, schema, backend);
|
|
@@ -28,6 +28,8 @@ export interface DreamdbBackend {
|
|
|
28
28
|
etag?: string;
|
|
29
29
|
size?: number;
|
|
30
30
|
}>;
|
|
31
|
+
/** Exact requested keys proven present; absence is never inferred from an incomplete listing. */
|
|
32
|
+
hasMany?(keys: readonly string[]): Promise<Set<string>>;
|
|
31
33
|
put(key: string, bytes: Uint8Array, opts?: {
|
|
32
34
|
ifMatch?: string;
|
|
33
35
|
ifNoneMatchStar?: boolean;
|
|
@@ -71,6 +73,13 @@ export declare class S3Backend {
|
|
|
71
73
|
etag?: string;
|
|
72
74
|
size?: number;
|
|
73
75
|
}>;
|
|
76
|
+
/**
|
|
77
|
+
* A bounded existence query for incremental chunks in THIS dataset. Published
|
|
78
|
+
* manifests are useful hints, not proof that their objects still exist.
|
|
79
|
+
* Never enumerate the dataset/history: at most four pages of the chunk prefix,
|
|
80
|
+
* then probe unresolved keys. Small/generic requests need only parallel HEADs.
|
|
81
|
+
*/
|
|
82
|
+
hasMany(keys: readonly string[]): Promise<Set<string>>;
|
|
74
83
|
/**
|
|
75
84
|
* Conditional PUT. DreamDB publishes refs with a compare-and-swap, so the
|
|
76
85
|
* precondition failures are expected control flow, not errors:
|
|
@@ -30,6 +30,65 @@ function unquote(etag) {
|
|
|
30
30
|
return undefined;
|
|
31
31
|
return etag.replace(/^"|"$/g, "");
|
|
32
32
|
}
|
|
33
|
+
const CHUNK_KEY = /^artifact-chunks\/v1\/[a-f0-9]{64}\.gz$/;
|
|
34
|
+
const CHUNK_PREFIX = 'artifact-chunks/v1/';
|
|
35
|
+
const EXISTENCE_CONCURRENCY = 8;
|
|
36
|
+
const EXISTENCE_LIST_PAGES = 4;
|
|
37
|
+
const EXISTENCE_LIST_BYTES = 2 * 1024 * 1024;
|
|
38
|
+
/** S3 XML scalar values, including opaque continuation tokens. No entity expansion. */
|
|
39
|
+
function xmlValue(xml, tag) {
|
|
40
|
+
const matches = [...xml.matchAll(new RegExp(`<${tag}>([^<]*)<\\/${tag}>`, 'g'))];
|
|
41
|
+
if (matches.length > 1)
|
|
42
|
+
throw new Error(`Invalid S3 listing: duplicate ${tag}`);
|
|
43
|
+
const value = matches[0]?.[1];
|
|
44
|
+
if (value === undefined)
|
|
45
|
+
return undefined;
|
|
46
|
+
if (/&(?!(?:amp|lt|gt|quot|apos|#\d+|#x[0-9a-fA-F]+);)/.test(value))
|
|
47
|
+
throw new Error('Invalid S3 XML entity');
|
|
48
|
+
return value.replace(/&(amp|lt|gt|quot|apos|#\d+|#x[0-9a-fA-F]+);/g, (_, entity) => {
|
|
49
|
+
const named = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'" };
|
|
50
|
+
if (named[entity] !== undefined)
|
|
51
|
+
return named[entity];
|
|
52
|
+
const point = entity.startsWith('#x') ? parseInt(entity.slice(2), 16) : Number(entity.slice(1));
|
|
53
|
+
if (!Number.isSafeInteger(point) || point <= 0 || point > 0x10ffff || (point >= 0xd800 && point <= 0xdfff))
|
|
54
|
+
throw new Error('Invalid S3 XML character');
|
|
55
|
+
return String.fromCodePoint(point);
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
async function boundedResponseText(response, limit) {
|
|
59
|
+
if (Number(response.headers.get('content-length')) > limit) {
|
|
60
|
+
await response.body?.cancel();
|
|
61
|
+
throw new Error('S3 listing response exceeds its size bound');
|
|
62
|
+
}
|
|
63
|
+
if (!response.body)
|
|
64
|
+
return '';
|
|
65
|
+
const reader = response.body.getReader();
|
|
66
|
+
const parts = [];
|
|
67
|
+
let length = 0;
|
|
68
|
+
try {
|
|
69
|
+
while (true) {
|
|
70
|
+
const result = await reader.read();
|
|
71
|
+
if (result.done)
|
|
72
|
+
break;
|
|
73
|
+
length += result.value.length;
|
|
74
|
+
if (length > limit) {
|
|
75
|
+
await reader.cancel();
|
|
76
|
+
throw new Error('S3 listing response exceeds its size bound');
|
|
77
|
+
}
|
|
78
|
+
parts.push(result.value);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
finally {
|
|
82
|
+
reader.releaseLock();
|
|
83
|
+
}
|
|
84
|
+
const bytes = new Uint8Array(length);
|
|
85
|
+
let offset = 0;
|
|
86
|
+
for (const part of parts) {
|
|
87
|
+
bytes.set(part, offset);
|
|
88
|
+
offset += part.length;
|
|
89
|
+
}
|
|
90
|
+
return new TextDecoder('utf-8', { fatal: true }).decode(bytes);
|
|
91
|
+
}
|
|
33
92
|
export class S3Backend {
|
|
34
93
|
client;
|
|
35
94
|
base;
|
|
@@ -78,6 +137,111 @@ export class S3Backend {
|
|
|
78
137
|
size: len ? Number(len) : undefined,
|
|
79
138
|
};
|
|
80
139
|
}
|
|
140
|
+
/**
|
|
141
|
+
* A bounded existence query for incremental chunks in THIS dataset. Published
|
|
142
|
+
* manifests are useful hints, not proof that their objects still exist.
|
|
143
|
+
* Never enumerate the dataset/history: at most four pages of the chunk prefix,
|
|
144
|
+
* then probe unresolved keys. Small/generic requests need only parallel HEADs.
|
|
145
|
+
*/
|
|
146
|
+
async hasMany(keys) {
|
|
147
|
+
const wanted = new Set(keys);
|
|
148
|
+
for (const key of wanted) {
|
|
149
|
+
if (!key || key.startsWith('/') || /[\\%?#\x00-\x1f\x7f]/.test(key) ||
|
|
150
|
+
key.split('/').some(segment => segment === '.' || segment === '..'))
|
|
151
|
+
throw new Error('Existence checks require canonical dataset-relative keys');
|
|
152
|
+
}
|
|
153
|
+
const found = new Set();
|
|
154
|
+
const probeRemaining = async () => {
|
|
155
|
+
const pending = [...wanted].filter(key => !found.has(key));
|
|
156
|
+
let next = 0;
|
|
157
|
+
let failure;
|
|
158
|
+
await Promise.all(Array.from({ length: Math.min(EXISTENCE_CONCURRENCY, pending.length) }, async () => {
|
|
159
|
+
while (next < pending.length && !failure) {
|
|
160
|
+
const key = pending[next++];
|
|
161
|
+
try {
|
|
162
|
+
const result = await this.head(key);
|
|
163
|
+
if (result.exists === true)
|
|
164
|
+
found.add(key);
|
|
165
|
+
else if (result.exists !== false)
|
|
166
|
+
throw new Error('Storage returned an indeterminate existence result');
|
|
167
|
+
}
|
|
168
|
+
catch (error) {
|
|
169
|
+
failure = error;
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
}));
|
|
173
|
+
if (failure)
|
|
174
|
+
throw failure;
|
|
175
|
+
return found;
|
|
176
|
+
};
|
|
177
|
+
if (wanted.size <= EXISTENCE_CONCURRENCY || [...wanted].some(key => !CHUNK_KEY.test(key)))
|
|
178
|
+
return probeRemaining();
|
|
179
|
+
const base = new URL(this.base);
|
|
180
|
+
const segments = base.pathname.replace(/^\/+|\/+$/g, '').split('/');
|
|
181
|
+
const bucket = segments.shift();
|
|
182
|
+
if (!bucket)
|
|
183
|
+
return probeRemaining();
|
|
184
|
+
const datasetPrefix = decodeURIComponent(segments.join('/'));
|
|
185
|
+
const relativePrefix = datasetPrefix ? `${datasetPrefix}/` : '';
|
|
186
|
+
const prefix = `${relativePrefix}${CHUNK_PREFIX}`;
|
|
187
|
+
const seenTokens = new Set();
|
|
188
|
+
let token;
|
|
189
|
+
for (let page = 0; page < EXISTENCE_LIST_PAGES; page++) {
|
|
190
|
+
const query = new URLSearchParams({ 'list-type': '2', prefix, 'max-keys': '1000', 'encoding-type': 'url' });
|
|
191
|
+
if (token)
|
|
192
|
+
query.set('continuation-token', token);
|
|
193
|
+
const response = await this.client.fetch(`${base.origin}/${bucket}?${query}`, { method: 'GET' });
|
|
194
|
+
const text = await boundedResponseText(response, response.ok ? EXISTENCE_LIST_BYTES : 64 * 1024);
|
|
195
|
+
if (!response.ok) {
|
|
196
|
+
const code = xmlValue(text, 'Code');
|
|
197
|
+
if (['ExpiredToken', 'InvalidToken', 'InvalidAccessKeyId', 'SignatureDoesNotMatch', 'TokenRefreshRequired', 'RequestExpired'].includes(code ?? ''))
|
|
198
|
+
throw new Error(`LIST artifact chunks → ${response.status} (${code})`);
|
|
199
|
+
// Some compatible stores deny listing but permit scoped object reads.
|
|
200
|
+
// Expired credentials, signature errors and transient failures propagate.
|
|
201
|
+
if ((response.status === 403 && (!code || code === 'AccessDenied')) ||
|
|
202
|
+
response.status === 405 || response.status === 501 ||
|
|
203
|
+
(response.status === 400 && ['NotImplemented', 'NotSupported', 'UnsupportedOperation'].includes(code ?? '')))
|
|
204
|
+
return probeRemaining();
|
|
205
|
+
throw new Error(`LIST artifact chunks → ${response.status}${code ? ` (${code})` : ''}`);
|
|
206
|
+
}
|
|
207
|
+
if (/<!DOCTYPE|<!ENTITY/i.test(text) || !/<ListBucketResult(?:\s[^>]*)?>/.test(text) || !/<\/ListBucketResult>\s*$/.test(text))
|
|
208
|
+
throw new Error('Invalid S3 listing response');
|
|
209
|
+
const encoding = xmlValue(text, 'EncodingType');
|
|
210
|
+
if (encoding !== undefined && encoding !== 'url')
|
|
211
|
+
throw new Error('Unsupported S3 listing encoding');
|
|
212
|
+
const decode = (value) => encoding === 'url' ? decodeURIComponent(value) : value;
|
|
213
|
+
if (decode(xmlValue(text, 'Prefix') ?? '') !== prefix)
|
|
214
|
+
throw new Error('S3 listing returned a different prefix');
|
|
215
|
+
const truncated = xmlValue(text, 'IsTruncated');
|
|
216
|
+
if (truncated !== 'true' && truncated !== 'false')
|
|
217
|
+
throw new Error('Invalid S3 listing truncation status');
|
|
218
|
+
const contents = [...text.matchAll(/<Contents>([\s\S]*?)<\/Contents>/g)];
|
|
219
|
+
if (contents.length > 1000 || /<Contents\s*\/>/.test(text) || (text.match(/<Contents(?:\s|>)/g) ?? []).length !== contents.length ||
|
|
220
|
+
(text.match(/<\/Contents>/g) ?? []).length !== contents.length)
|
|
221
|
+
throw new Error('Invalid S3 listing contents/page bound');
|
|
222
|
+
const keyCount = xmlValue(text, 'KeyCount');
|
|
223
|
+
if (keyCount !== undefined && (!/^\d+$/.test(keyCount) || Number(keyCount) !== contents.length))
|
|
224
|
+
throw new Error('S3 listing key count mismatch');
|
|
225
|
+
for (const entry of contents) {
|
|
226
|
+
const raw = xmlValue(entry[1], 'Key');
|
|
227
|
+
if (raw === undefined)
|
|
228
|
+
throw new Error('S3 listing entry is missing its key');
|
|
229
|
+
const key = decode(raw);
|
|
230
|
+
if (!key.startsWith(prefix))
|
|
231
|
+
throw new Error('S3 listing returned a key outside its prefix');
|
|
232
|
+
const relative = key.slice(relativePrefix.length);
|
|
233
|
+
if (wanted.has(relative))
|
|
234
|
+
found.add(relative);
|
|
235
|
+
}
|
|
236
|
+
if (found.size === wanted.size || truncated === 'false')
|
|
237
|
+
return found;
|
|
238
|
+
token = xmlValue(text, 'NextContinuationToken');
|
|
239
|
+
if (!token || token.length > 16 * 1024 || seenTokens.has(token))
|
|
240
|
+
throw new Error('Invalid S3 listing continuation token');
|
|
241
|
+
seenTokens.add(token);
|
|
242
|
+
}
|
|
243
|
+
return probeRemaining();
|
|
244
|
+
}
|
|
81
245
|
/**
|
|
82
246
|
* Conditional PUT. DreamDB publishes refs with a compare-and-swap, so the
|
|
83
247
|
* precondition failures are expected control flow, not errors:
|
package/dist/cli/config.d.ts
CHANGED
|
@@ -8,8 +8,8 @@ export declare function resolveToken(flag?: string, dir?: string): string | null
|
|
|
8
8
|
/** Decode a JWT payload without verifying the signature. Returns {} on failure. */
|
|
9
9
|
export declare function decodeJwtPayload(token: string): Record<string, unknown>;
|
|
10
10
|
/**
|
|
11
|
-
* Resolve the
|
|
12
|
-
* explicit
|
|
11
|
+
* Resolve a command's default namespace without changing the signed-in user's identity:
|
|
12
|
+
* explicit → selected CLI namespace → personal cache → GET /auth/me → JWT decode.
|
|
13
13
|
*
|
|
14
14
|
* The cache is only trusted when the effective (remote, token) is exactly
|
|
15
15
|
* the materialized pair — under a flag or env-var override it may describe
|
package/dist/cli/config.js
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
1
2
|
import { identityMode, loadMachineConfig } from "./auth/machine.js";
|
|
2
|
-
// Shared connection config —
|
|
3
|
-
//
|
|
3
|
+
// Shared connection config — token and remote match dreamlake-py.
|
|
4
|
+
// Namespace selection is CLI-local; SDK domain methods take explicit scopes:
|
|
4
5
|
//
|
|
5
6
|
// token: flag → DREAMLAKE_API_KEY → ~/.dreamlake/tokens.json → null
|
|
6
7
|
// remote: flag → DREAMLAKE_REMOTE → config.json remote_url → prod
|
|
7
|
-
// namespace: explicit
|
|
8
|
+
// namespace: explicit → selected CLI namespace → personal cache → GET /auth/me → JWT decode
|
|
8
9
|
//
|
|
9
10
|
// The multi-env cache (auth.yml) is deliberately absent here — it is a
|
|
10
11
|
// write-path convenience for `env use` and is never consulted to answer
|
|
@@ -56,8 +57,8 @@ export function decodeJwtPayload(token) {
|
|
|
56
57
|
}
|
|
57
58
|
}
|
|
58
59
|
/**
|
|
59
|
-
* Resolve the
|
|
60
|
-
* explicit
|
|
60
|
+
* Resolve a command's default namespace without changing the signed-in user's identity:
|
|
61
|
+
* explicit → selected CLI namespace → personal cache → GET /auth/me → JWT decode.
|
|
61
62
|
*
|
|
62
63
|
* The cache is only trusted when the effective (remote, token) is exactly
|
|
63
64
|
* the materialized pair — under a flag or env-var override it may describe
|
|
@@ -67,6 +68,13 @@ export async function resolveNamespace(explicit, opts) {
|
|
|
67
68
|
if (explicit)
|
|
68
69
|
return explicit;
|
|
69
70
|
const cfg = readConfig(opts.dir);
|
|
71
|
+
const selected = cfg.default_namespace;
|
|
72
|
+
if (selected && typeof selected === 'object' && selected.remote === stripTrailingSlash(opts.remote) &&
|
|
73
|
+
selected.tokenSha256 === createHash('sha256').update(opts.token).digest('hex')) {
|
|
74
|
+
if (typeof selected.namespace !== 'string' || !/^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$/.test(selected.namespace))
|
|
75
|
+
throw new Error('Saved CLI namespace is invalid; run dreamlake su <namespace> again');
|
|
76
|
+
return selected.namespace;
|
|
77
|
+
}
|
|
70
78
|
if (cfg.namespace &&
|
|
71
79
|
cfg.remote_url &&
|
|
72
80
|
stripTrailingSlash(cfg.remote_url) === stripTrailingSlash(opts.remote) &&
|
|
@@ -12,6 +12,7 @@ export const helpExamples = {
|
|
|
12
12
|
"runnable execute": "dreamlake runnable execute exec/acme/recipes.process@EXECUTION_DIGEST \\\n --queue cpu --inputs record.json --timeout-seconds 600 --json\ndreamlake runnable execute exec/acme/recipes.process@EXECUTION_DIGEST \\\n --queue cpu --inputs record.json --timeout-seconds 600 --attempt 2 --json",
|
|
13
13
|
"runnable execution-receipt": "dreamlake runnable execution-receipt exec/acme/recipes.process@EXECUTION_DIGEST \\\n --queue cpu --inputs record.json --timeout-seconds 600 --wait-seconds 30",
|
|
14
14
|
"hosts": "dreamlake hosts\ndreamlake hosts --all\ndreamlake hosts -h\ndreamlake hosts list -h",
|
|
15
|
+
"su": "dreamlake su fortyfive\ndreamlake hosts list\ndreamlake hosts list --namespace geyang\ndreamlake su geyang",
|
|
15
16
|
"nymph": "dreamlake nymph\ndreamlake nymph --json",
|
|
16
17
|
"nymph setup": "dreamlake nymph setup --name alice/computers/laptop --lakeshore-id CONNECTION_ID",
|
|
17
18
|
"nymph config": "dreamlake nymph config --name alice/computers/laptop",
|