@decocms/blocks 8.1.2 → 8.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/protocol/client.d.ts +17 -4
- package/dist/protocol/client.d.ts.map +1 -1
- package/dist/protocol/client.js +46 -10
- package/dist/protocol/client.js.map +1 -1
- package/dist/protocol/conformance/cases/changes.d.ts +3 -0
- package/dist/protocol/conformance/cases/changes.d.ts.map +1 -0
- package/dist/protocol/conformance/cases/changes.js +295 -0
- package/dist/protocol/conformance/cases/changes.js.map +1 -0
- package/dist/protocol/conformance/context.d.ts +14 -0
- package/dist/protocol/conformance/context.d.ts.map +1 -1
- package/dist/protocol/conformance/context.js.map +1 -1
- package/dist/protocol/conformance/index.d.ts.map +1 -1
- package/dist/protocol/conformance/index.js +4 -1
- package/dist/protocol/conformance/index.js.map +1 -1
- package/dist/protocol/errors.d.ts +26 -0
- package/dist/protocol/errors.d.ts.map +1 -1
- package/dist/protocol/errors.js.map +1 -1
- package/dist/protocol/index.d.ts +4 -2
- package/dist/protocol/index.d.ts.map +1 -1
- package/dist/protocol/index.js +4 -2
- package/dist/protocol/index.js.map +1 -1
- package/dist/protocol/params.d.ts +4 -2
- package/dist/protocol/params.d.ts.map +1 -1
- package/dist/protocol/params.js +11 -1
- package/dist/protocol/params.js.map +1 -1
- package/dist/protocol/server/assets.d.ts +1 -1
- package/dist/protocol/server/assets.js +1 -1
- package/dist/protocol/server/core.d.ts +12 -1
- package/dist/protocol/server/core.d.ts.map +1 -1
- package/dist/protocol/server/core.js +8 -0
- package/dist/protocol/server/core.js.map +1 -1
- package/dist/protocol/server/index.d.ts +1 -1
- package/dist/protocol/server/index.js +1 -1
- package/dist/protocol/server/methods/apply.d.ts +37 -0
- package/dist/protocol/server/methods/apply.d.ts.map +1 -1
- package/dist/protocol/server/methods/apply.js +4 -3
- package/dist/protocol/server/methods/apply.js.map +1 -1
- package/dist/protocol/server/methods/changes.d.ts +6 -0
- package/dist/protocol/server/methods/changes.d.ts.map +1 -0
- package/dist/protocol/server/methods/changes.js +381 -0
- package/dist/protocol/server/methods/changes.js.map +1 -0
- package/dist/protocol/server/rpc.d.ts.map +1 -1
- package/dist/protocol/server/rpc.js +7 -0
- package/dist/protocol/server/rpc.js.map +1 -1
- package/dist/protocol/storage/fs/git.d.ts +51 -0
- package/dist/protocol/storage/fs/git.d.ts.map +1 -0
- package/dist/protocol/storage/fs/git.js +274 -0
- package/dist/protocol/storage/fs/git.js.map +1 -0
- package/dist/protocol/storage/fs/index.d.ts +2 -0
- package/dist/protocol/storage/fs/index.d.ts.map +1 -1
- package/dist/protocol/storage/fs/index.js +34 -0
- package/dist/protocol/storage/fs/index.js.map +1 -1
- package/dist/protocol/storage/memory.d.ts +10 -0
- package/dist/protocol/storage/memory.d.ts.map +1 -1
- package/dist/protocol/storage/memory.js +45 -0
- package/dist/protocol/storage/memory.js.map +1 -1
- package/dist/protocol/storage.d.ts +49 -1
- package/dist/protocol/storage.d.ts.map +1 -1
- package/dist/protocol/storage.js.map +1 -1
- package/dist/protocol/types.d.ts +105 -4
- package/dist/protocol/types.d.ts.map +1 -1
- package/dist/protocol/types.js +16 -3
- package/dist/protocol/types.js.map +1 -1
- package/dist/v8/cli/run.d.ts.map +1 -1
- package/dist/v8/cli/run.js +2 -1
- package/dist/v8/cli/run.js.map +1 -1
- package/dist/v8/cli/serve/server.d.ts +2 -0
- package/dist/v8/cli/serve/server.d.ts.map +1 -1
- package/dist/v8/cli/serve/server.js +37 -2
- package/dist/v8/cli/serve/server.js.map +1 -1
- package/package.json +1 -1
- package/src/protocol/client.ts +81 -13
- package/src/protocol/conformance/cases/changes.ts +362 -0
- package/src/protocol/conformance/context.ts +12 -0
- package/src/protocol/conformance/index.ts +4 -1
- package/src/protocol/errors.ts +17 -0
- package/src/protocol/index.ts +4 -2
- package/src/protocol/params.ts +26 -3
- package/src/protocol/server/assets.ts +1 -1
- package/src/protocol/server/core.ts +9 -0
- package/src/protocol/server/index.ts +1 -1
- package/src/protocol/server/methods/apply.ts +8 -4
- package/src/protocol/server/methods/changes.ts +489 -0
- package/src/protocol/server/rpc.ts +7 -0
- package/src/protocol/storage/fs/git.ts +307 -0
- package/src/protocol/storage/fs/index.ts +40 -0
- package/src/protocol/storage/memory.ts +57 -0
- package/src/protocol/storage.ts +51 -1
- package/src/protocol/types.ts +106 -4
- package/src/v8/cli/run.ts +2 -1
- package/src/v8/cli/serve/server.ts +46 -2
|
@@ -0,0 +1,307 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The filesystem storage's base: the git `HEAD` of the repository the app
|
|
3
|
+
* root is in. Read-only: it runs `git` (no new dependency, no shell, argv
|
|
4
|
+
* arrays only) and never writes git state — no index writes, no commits, no
|
|
5
|
+
* `git restore`. A discard writes the files themselves, through the storage's
|
|
6
|
+
* own commit.
|
|
7
|
+
*
|
|
8
|
+
* Changes are the working tree against `HEAD` (the index is never
|
|
9
|
+
* consulted), with git's own idea of equal: a file whose bytes differ only by
|
|
10
|
+
* the checkout filters (`eol=crlf`, `autocrlf`, LFS) isn't a change.
|
|
11
|
+
*/
|
|
12
|
+
import { execFile } from "node:child_process";
|
|
13
|
+
import { join } from "node:path";
|
|
14
|
+
import { isBlockFileName } from "../../keys.ts";
|
|
15
|
+
import { type StorageBase, type StorageFile, StorageUnavailableError } from "../../storage.ts";
|
|
16
|
+
|
|
17
|
+
/** The base revision reported for a repository with no commits yet (an empty base). */
|
|
18
|
+
export const UNBORN_REVISION = "unborn";
|
|
19
|
+
|
|
20
|
+
/** Environment that would point git at another repository, index or object store. */
|
|
21
|
+
const INHERITED_GIT_ENV = [
|
|
22
|
+
"GIT_DIR",
|
|
23
|
+
"GIT_WORK_TREE",
|
|
24
|
+
"GIT_INDEX_FILE",
|
|
25
|
+
"GIT_OBJECT_DIRECTORY",
|
|
26
|
+
"GIT_ALTERNATE_OBJECT_DIRECTORIES",
|
|
27
|
+
"GIT_NAMESPACE",
|
|
28
|
+
"GIT_PREFIX",
|
|
29
|
+
"GIT_COMMON_DIR",
|
|
30
|
+
];
|
|
31
|
+
|
|
32
|
+
const COMMIT_ID = /^[0-9a-f]{40,64}$/;
|
|
33
|
+
|
|
34
|
+
/** Files per `git hash-object` call, so an argv stays far below any OS limit. */
|
|
35
|
+
const HASH_CHUNK = 100;
|
|
36
|
+
|
|
37
|
+
/** Base bodies read at once. */
|
|
38
|
+
const READ_CONCURRENCY = 4;
|
|
39
|
+
|
|
40
|
+
export interface GitOptions {
|
|
41
|
+
/** The git binary (default `git`, from `PATH`). */
|
|
42
|
+
git?: string;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/** A git call that failed: `missing` when the binary isn't there. */
|
|
46
|
+
export class GitError extends Error {
|
|
47
|
+
constructor(
|
|
48
|
+
message: string,
|
|
49
|
+
readonly reason: "missing" | "failed",
|
|
50
|
+
readonly exitCode: number | null,
|
|
51
|
+
readonly stderr: string,
|
|
52
|
+
) {
|
|
53
|
+
super(message);
|
|
54
|
+
this.name = "GitError";
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function gitEnv(): NodeJS.ProcessEnv {
|
|
59
|
+
const env: NodeJS.ProcessEnv = { ...process.env };
|
|
60
|
+
for (const key of INHERITED_GIT_ENV) delete env[key];
|
|
61
|
+
return {
|
|
62
|
+
...env,
|
|
63
|
+
// Pathspecs are literal: a file named `:(glob)x.json` is just a file.
|
|
64
|
+
GIT_LITERAL_PATHSPECS: "1",
|
|
65
|
+
// Never refresh (write) the index while reading.
|
|
66
|
+
GIT_OPTIONAL_LOCKS: "0",
|
|
67
|
+
GIT_TERMINAL_PROMPT: "0",
|
|
68
|
+
LC_ALL: "C",
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/** Runs git and resolves with its stdout bytes; rejects with a `GitError`. */
|
|
73
|
+
export function runGit(cwd: string, args: string[], options: GitOptions = {}): Promise<Buffer> {
|
|
74
|
+
const git = options.git ?? "git";
|
|
75
|
+
return new Promise((resolve, reject) => {
|
|
76
|
+
execFile(
|
|
77
|
+
git,
|
|
78
|
+
args,
|
|
79
|
+
{
|
|
80
|
+
cwd,
|
|
81
|
+
env: gitEnv(),
|
|
82
|
+
encoding: "buffer",
|
|
83
|
+
maxBuffer: 64 * 1024 * 1024,
|
|
84
|
+
timeout: 30_000,
|
|
85
|
+
windowsHide: true,
|
|
86
|
+
},
|
|
87
|
+
(error, stdout, stderr) => {
|
|
88
|
+
if (!error) return resolve(stdout);
|
|
89
|
+
const code = (error as NodeJS.ErrnoException).code;
|
|
90
|
+
const text = Buffer.isBuffer(stderr) ? stderr.toString("utf8") : String(stderr ?? "");
|
|
91
|
+
if (code === "ENOENT") {
|
|
92
|
+
return reject(new GitError(`${git} isn't installed`, "missing", null, text));
|
|
93
|
+
}
|
|
94
|
+
const exitCode = typeof code === "number" ? code : null;
|
|
95
|
+
const line = text.split("\n").find((l) => l.trim() !== "") ?? error.message;
|
|
96
|
+
reject(new GitError(line.trim(), "failed", exitCode, text));
|
|
97
|
+
},
|
|
98
|
+
);
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
const firstLine = (bytes: Buffer) => bytes.toString("utf8").split("\n")[0]!.trim();
|
|
103
|
+
|
|
104
|
+
/** Where `dir` stands in git: inside a work tree, not, or git is missing. */
|
|
105
|
+
export type GitWorkTreeCheck =
|
|
106
|
+
| { status: "ok" }
|
|
107
|
+
| { status: "missing" }
|
|
108
|
+
| { status: "not-a-repository" }
|
|
109
|
+
| { status: "failed"; message: string };
|
|
110
|
+
|
|
111
|
+
/** Checks that `dir` is inside a git work tree (not a bare repository, not inside `.git`). */
|
|
112
|
+
export async function checkGitWorkTree(
|
|
113
|
+
dir: string,
|
|
114
|
+
options: GitOptions = {},
|
|
115
|
+
): Promise<GitWorkTreeCheck> {
|
|
116
|
+
try {
|
|
117
|
+
const inside = firstLine(await runGit(dir, ["rev-parse", "--is-inside-work-tree"], options));
|
|
118
|
+
return inside === "true" ? { status: "ok" } : { status: "not-a-repository" };
|
|
119
|
+
} catch (error) {
|
|
120
|
+
if (!(error instanceof GitError)) throw error;
|
|
121
|
+
if (error.reason === "missing") return { status: "missing" };
|
|
122
|
+
if (/not a git repository/i.test(error.stderr)) return { status: "not-a-repository" };
|
|
123
|
+
return { status: "failed", message: error.message };
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
function unavailable(error: unknown): never {
|
|
128
|
+
if (error instanceof GitError) throw new StorageUnavailableError(`git: ${error.message}`);
|
|
129
|
+
throw error;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/** A blob of the base: the file name inside `.deco/blocks` and its blob id. */
|
|
133
|
+
type BaseBlob = StorageFile;
|
|
134
|
+
|
|
135
|
+
/** The git side of one app root: its base, what differs from it, and base bodies. */
|
|
136
|
+
export interface GitBase {
|
|
137
|
+
base(): Promise<StorageBase>;
|
|
138
|
+
/** The saved-block files in the base, with their blob ids as versions. */
|
|
139
|
+
baseFiles(base: StorageBase): Promise<BaseBlob[]>;
|
|
140
|
+
/**
|
|
141
|
+
* The working-tree files (by name, with their raw blob hash) that differ
|
|
142
|
+
* from `baseFiles`: missing on one side, or different after git's filters.
|
|
143
|
+
*/
|
|
144
|
+
differing(base: StorageBase, current: StorageFile[]): Promise<string[]>;
|
|
145
|
+
/** Base bodies as a checkout writes them (smudge and eol filters applied). */
|
|
146
|
+
readBase(base: StorageBase, files: string[]): Promise<Map<string, Buffer>>;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/**
|
|
150
|
+
* The git base of the app root `root` (the folder that contains `.deco`),
|
|
151
|
+
* whose saved blocks are `.deco/blocks/*.json`.
|
|
152
|
+
*/
|
|
153
|
+
export function createGitBase(root: string, options: GitOptions = {}): GitBase {
|
|
154
|
+
const blocksDir = join(root, ".deco", "blocks");
|
|
155
|
+
let prefix: Promise<string> | undefined;
|
|
156
|
+
/** Base listings by commit: a commit's tree never changes. */
|
|
157
|
+
let listing: { revision: string; files: Promise<BaseBlob[]> } | undefined;
|
|
158
|
+
|
|
159
|
+
const git = (args: string[], cwd = root) => runGit(cwd, args, options);
|
|
160
|
+
|
|
161
|
+
async function base(): Promise<StorageBase> {
|
|
162
|
+
try {
|
|
163
|
+
let revision = UNBORN_REVISION;
|
|
164
|
+
try {
|
|
165
|
+
const sha = firstLine(await git(["rev-parse", "-q", "--verify", "HEAD^{commit}"]));
|
|
166
|
+
if (COMMIT_ID.test(sha)) revision = sha;
|
|
167
|
+
} catch (error) {
|
|
168
|
+
// `--verify -q` exits 1 without output on an unborn HEAD (a fresh `git init`).
|
|
169
|
+
if (!(error instanceof GitError) || error.reason !== "failed" || error.stderr.trim()) {
|
|
170
|
+
throw error;
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
let resolvedRef: string | null = null;
|
|
174
|
+
try {
|
|
175
|
+
resolvedRef = firstLine(await git(["symbolic-ref", "-q", "--short", "HEAD"])) || null;
|
|
176
|
+
} catch (error) {
|
|
177
|
+
// Exits 1 on a detached HEAD.
|
|
178
|
+
if (!(error instanceof GitError) || error.reason !== "failed" || error.stderr.trim()) {
|
|
179
|
+
throw error;
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
return { revision, resolvedRef };
|
|
183
|
+
} catch (error) {
|
|
184
|
+
unavailable(error);
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
async function listBase(revision: string): Promise<BaseBlob[]> {
|
|
189
|
+
if (!COMMIT_ID.test(revision)) return [];
|
|
190
|
+
// Relative to the app root (cwd): `.deco/blocks/<file>`. Only regular
|
|
191
|
+
// files directly in the folder; subfolders, symlinks and submodules aren't blocks.
|
|
192
|
+
const out = await git(["ls-tree", "-z", "--long", revision, "--", ".deco/blocks/"]);
|
|
193
|
+
const files: BaseBlob[] = [];
|
|
194
|
+
for (const record of out.toString("utf8").split("\0")) {
|
|
195
|
+
const tab = record.indexOf("\t");
|
|
196
|
+
if (tab === -1) continue;
|
|
197
|
+
const [mode, type, id, size] = record.slice(0, tab).trim().split(/\s+/);
|
|
198
|
+
const path = record.slice(tab + 1);
|
|
199
|
+
if (type !== "blob" || (mode !== "100644" && mode !== "100755")) continue;
|
|
200
|
+
if (!path.startsWith(".deco/blocks/")) continue;
|
|
201
|
+
const file = path.slice(".deco/blocks/".length);
|
|
202
|
+
if (!isBlockFileName(file) || !id || !COMMIT_ID.test(id)) continue;
|
|
203
|
+
const bytes = Number(size);
|
|
204
|
+
files.push(
|
|
205
|
+
Number.isFinite(bytes) ? { file, version: id, size: bytes } : { file, version: id },
|
|
206
|
+
);
|
|
207
|
+
}
|
|
208
|
+
return files;
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
function baseFiles(current: StorageBase): Promise<BaseBlob[]> {
|
|
212
|
+
if (listing?.revision !== current.revision) {
|
|
213
|
+
const files = listBase(current.revision).catch(unavailable);
|
|
214
|
+
listing = { revision: current.revision, files };
|
|
215
|
+
files.catch(() => {
|
|
216
|
+
if (listing?.files === files) listing = undefined;
|
|
217
|
+
});
|
|
218
|
+
}
|
|
219
|
+
return listing.files;
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
/** The blob ids git would store for these working-tree files (filters applied). */
|
|
223
|
+
async function hashWorking(files: string[]): Promise<string[]> {
|
|
224
|
+
const ids: string[] = [];
|
|
225
|
+
for (let i = 0; i < files.length; i += HASH_CHUNK) {
|
|
226
|
+
const chunk = files.slice(i, i + HASH_CHUNK);
|
|
227
|
+
const out = await git(["hash-object", "--", ...chunk], blocksDir);
|
|
228
|
+
const lines = out.toString("utf8").split("\n").filter(Boolean);
|
|
229
|
+
if (lines.length !== chunk.length) {
|
|
230
|
+
throw new GitError("hash-object answered an unexpected listing", "failed", null, "");
|
|
231
|
+
}
|
|
232
|
+
ids.push(...lines);
|
|
233
|
+
}
|
|
234
|
+
return ids;
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
async function differing(current: StorageBase, now: StorageFile[]): Promise<string[]> {
|
|
238
|
+
try {
|
|
239
|
+
const inBase = new Map((await baseFiles(current)).map((f) => [f.file, f.version]));
|
|
240
|
+
const changed = new Set<string>();
|
|
241
|
+
const toConfirm: string[] = [];
|
|
242
|
+
const present = new Set<string>();
|
|
243
|
+
for (const file of now) {
|
|
244
|
+
present.add(file.file);
|
|
245
|
+
const id = inBase.get(file.file);
|
|
246
|
+
if (id === undefined) changed.add(file.file);
|
|
247
|
+
// The raw bytes' hash equals the blob id: the same content, no filter involved.
|
|
248
|
+
else if (id !== file.version) toConfirm.push(file.file);
|
|
249
|
+
}
|
|
250
|
+
for (const file of inBase.keys()) if (!present.has(file)) changed.add(file);
|
|
251
|
+
if (toConfirm.length > 0) {
|
|
252
|
+
let ids: string[];
|
|
253
|
+
try {
|
|
254
|
+
ids = await hashWorking(toConfirm);
|
|
255
|
+
} catch (error) {
|
|
256
|
+
// A file vanished meanwhile: count every candidate as changed; the
|
|
257
|
+
// core's next snapshot reports the truth.
|
|
258
|
+
if (
|
|
259
|
+
error instanceof GitError &&
|
|
260
|
+
/No such file|cannot open|unable to/i.test(error.stderr)
|
|
261
|
+
) {
|
|
262
|
+
ids = toConfirm.map(() => "");
|
|
263
|
+
} else throw error;
|
|
264
|
+
}
|
|
265
|
+
toConfirm.forEach((file, i) => {
|
|
266
|
+
if (ids[i] !== inBase.get(file)) changed.add(file);
|
|
267
|
+
});
|
|
268
|
+
}
|
|
269
|
+
return [...changed].sort();
|
|
270
|
+
} catch (error) {
|
|
271
|
+
unavailable(error);
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
async function readBase(current: StorageBase, files: string[]): Promise<Map<string, Buffer>> {
|
|
276
|
+
try {
|
|
277
|
+
const out = new Map<string, Buffer>();
|
|
278
|
+
if (files.length === 0) return out;
|
|
279
|
+
const inBase = new Map((await baseFiles(current)).map((f) => [f.file, f.version]));
|
|
280
|
+
prefix ??= git(["rev-parse", "--show-prefix"]).then((b) => firstLine(b));
|
|
281
|
+
const repoPrefix = await prefix;
|
|
282
|
+
const wanted = files.filter((file) => isBlockFileName(file) && inBase.has(file));
|
|
283
|
+
for (let i = 0; i < wanted.length; i += READ_CONCURRENCY) {
|
|
284
|
+
await Promise.all(
|
|
285
|
+
wanted.slice(i, i + READ_CONCURRENCY).map(async (file) => {
|
|
286
|
+
const id = inBase.get(file)!;
|
|
287
|
+
// Per file: `cat-file --batch --filters` frames its output with the
|
|
288
|
+
// unfiltered size, so a filtered stream can't be split reliably.
|
|
289
|
+
const body = await git([
|
|
290
|
+
"cat-file",
|
|
291
|
+
"--filters",
|
|
292
|
+
`--path=${repoPrefix}.deco/blocks/${file}`,
|
|
293
|
+
id,
|
|
294
|
+
]);
|
|
295
|
+
out.set(file, body);
|
|
296
|
+
}),
|
|
297
|
+
);
|
|
298
|
+
}
|
|
299
|
+
return out;
|
|
300
|
+
} catch (error) {
|
|
301
|
+
prefix = undefined;
|
|
302
|
+
unavailable(error);
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
return { base, baseFiles, differing, readBase };
|
|
307
|
+
}
|
|
@@ -10,6 +10,12 @@
|
|
|
10
10
|
* - Uploads go to `public/assets` (or `assetsDir`), never overwriting a file.
|
|
11
11
|
*
|
|
12
12
|
* A file's version is its git blob hash, as `git hash-object` prints it.
|
|
13
|
+
*
|
|
14
|
+
* Its base (what `changes.*` compares with) is the git `HEAD` of the
|
|
15
|
+
* repository the app root is in, read with the `git` binary (./git.ts):
|
|
16
|
+
* `changes.list`, `changes.get` and `changes.discard` need the app root to be
|
|
17
|
+
* inside a git work tree, and answer Unavailable otherwise. `deco serve`
|
|
18
|
+
* refuses to start outside one. The other methods don't need git.
|
|
13
19
|
* Commits are serialized per `.deco` folder (in process and across processes)
|
|
14
20
|
* and applied with staged files and atomic renames, rolled back on failure,
|
|
15
21
|
* with the folder fsynced before a commit returns.
|
|
@@ -35,6 +41,7 @@ import {
|
|
|
35
41
|
type StoredFileBody,
|
|
36
42
|
type StoredSchema,
|
|
37
43
|
} from "../../storage.ts";
|
|
44
|
+
import { createGitBase } from "./git.ts";
|
|
38
45
|
import { gitBlobHash, revisionOf } from "./hash.ts";
|
|
39
46
|
import { withCommitLock } from "./lock.ts";
|
|
40
47
|
import { applyFileChange, sweepStaleTransactions } from "./transaction.ts";
|
|
@@ -58,6 +65,8 @@ export interface FsStorageOptions {
|
|
|
58
65
|
pollIntervalMs?: number;
|
|
59
66
|
/** How long a commit waits for another process's lock (default 10 s). */
|
|
60
67
|
lockTimeoutMs?: number;
|
|
68
|
+
/** The git binary the base is read with (default `git`, from `PATH`). */
|
|
69
|
+
git?: string;
|
|
61
70
|
}
|
|
62
71
|
|
|
63
72
|
export type FsStorage = ContentStorage & {
|
|
@@ -134,6 +143,7 @@ export function createFsStorage(options: FsStorageOptions): FsStorage {
|
|
|
134
143
|
const lockOptions = { timeoutMs: options.lockTimeoutMs ?? 10_000, staleMs: 30_000 };
|
|
135
144
|
let repoRoot: Promise<string> | undefined;
|
|
136
145
|
let swept = false;
|
|
146
|
+
const gitBase = createGitBase(root, { git: options.git });
|
|
137
147
|
|
|
138
148
|
/** file -> stat fingerprint and its hash, so a poll only rehashes changed files. */
|
|
139
149
|
const hashCache = new Map<string, { fingerprint: string; version: string }>();
|
|
@@ -279,6 +289,12 @@ export function createFsStorage(options: FsStorageOptions): FsStorage {
|
|
|
279
289
|
await sweepStaleTransactions(decoDir);
|
|
280
290
|
swept = true;
|
|
281
291
|
}
|
|
292
|
+
if (
|
|
293
|
+
attempt.expectedBase !== undefined &&
|
|
294
|
+
(await gitBase.base()).revision !== attempt.expectedBase
|
|
295
|
+
) {
|
|
296
|
+
return { status: "stale" } as const;
|
|
297
|
+
}
|
|
282
298
|
if (attempt.expectedSchemaVersion !== undefined) {
|
|
283
299
|
const schema = await readSchema();
|
|
284
300
|
if ((schema?.version ?? null) !== attempt.expectedSchemaVersion) {
|
|
@@ -300,6 +316,30 @@ export function createFsStorage(options: FsStorageOptions): FsStorage {
|
|
|
300
316
|
}
|
|
301
317
|
},
|
|
302
318
|
|
|
319
|
+
base: () => gitBase.base(),
|
|
320
|
+
|
|
321
|
+
async changes(base, current) {
|
|
322
|
+
const [files, baseFiles] = await Promise.all([
|
|
323
|
+
gitBase.differing(base, current.files),
|
|
324
|
+
gitBase.baseFiles(base),
|
|
325
|
+
]);
|
|
326
|
+
return { files, baseFiles };
|
|
327
|
+
},
|
|
328
|
+
|
|
329
|
+
async readBaseFiles(base, files) {
|
|
330
|
+
for (const file of files) assertBlockFile(file);
|
|
331
|
+
const versions = new Map((await gitBase.baseFiles(base)).map((f) => [f.file, f.version]));
|
|
332
|
+
const bodies = await gitBase.readBase(base, files);
|
|
333
|
+
// Exactly as a checkout writes them, a leading BOM included: a discard
|
|
334
|
+
// writes this text back, and anything else would still differ from HEAD.
|
|
335
|
+
const decoder = new TextDecoder("utf-8", { ignoreBOM: true });
|
|
336
|
+
const out: Record<string, StoredFileBody> = Object.create(null);
|
|
337
|
+
for (const [file, bytes] of bodies) {
|
|
338
|
+
out[file] = { text: decoder.decode(bytes), version: versions.get(file)! };
|
|
339
|
+
}
|
|
340
|
+
return out;
|
|
341
|
+
},
|
|
342
|
+
|
|
303
343
|
async putAsset(name, body) {
|
|
304
344
|
if (readOnly) throw new StorageUnavailableError("the storage is read-only");
|
|
305
345
|
if (!name || name.includes("/") || name.includes("\\") || name.startsWith(".")) {
|
|
@@ -32,6 +32,12 @@ export interface MemoryStorageState {
|
|
|
32
32
|
assets: Record<string, string>;
|
|
33
33
|
/** `false` simulates an app root with no `.deco` folder at all. */
|
|
34
34
|
hasDecoFolder: boolean;
|
|
35
|
+
/** The base's saved-block files (what `changes.*` compares with), by file name. Default: none. */
|
|
36
|
+
base: Record<string, string>;
|
|
37
|
+
/** The base's revision; moves with every `setBase`/`commitBase`. */
|
|
38
|
+
baseRevision: string;
|
|
39
|
+
/** The base's branch. Default `null`. */
|
|
40
|
+
baseRef: string | null;
|
|
35
41
|
}
|
|
36
42
|
|
|
37
43
|
export interface MemoryStorageOptions {
|
|
@@ -51,6 +57,10 @@ export interface MemoryStorage extends ContentStorage {
|
|
|
51
57
|
setFile(file: string, content: string | null): void;
|
|
52
58
|
/** Replaces `schema.gen.json` (`null` removes it). */
|
|
53
59
|
setSchema(schema: string | object | null): void;
|
|
60
|
+
/** Replaces the base's files, as a publish (or an outside commit) would. */
|
|
61
|
+
setBase(files: Record<string, string>): void;
|
|
62
|
+
/** Makes the current files the base, as a commit of every change would. */
|
|
63
|
+
commitBase(): void;
|
|
54
64
|
/** How many commits landed. */
|
|
55
65
|
readonly commits: number;
|
|
56
66
|
}
|
|
@@ -65,8 +75,13 @@ export function createMemoryStorage(options: MemoryStorageOptions = {}): MemoryS
|
|
|
65
75
|
secretsPublicKey: null,
|
|
66
76
|
assets: {},
|
|
67
77
|
hasDecoFolder: true,
|
|
78
|
+
base: {},
|
|
79
|
+
baseRevision: "base-0",
|
|
80
|
+
baseRef: null,
|
|
68
81
|
...options.state,
|
|
69
82
|
});
|
|
83
|
+
// structuredClone keeps shared references: the base is never the files object.
|
|
84
|
+
state.base = { ...state.base };
|
|
70
85
|
const description: StorageDescription = {
|
|
71
86
|
kind: "git",
|
|
72
87
|
root: ".",
|
|
@@ -107,6 +122,9 @@ export function createMemoryStorage(options: MemoryStorageOptions = {}): MemoryS
|
|
|
107
122
|
if (!isBlockFileName(file)) throw new StorageInvalidFileError(file);
|
|
108
123
|
}
|
|
109
124
|
if (!state.hasDecoFolder) throw new StorageNotFoundError("no .deco folder");
|
|
125
|
+
if (attempt.expectedBase !== undefined && attempt.expectedBase !== state.baseRevision) {
|
|
126
|
+
return { status: "stale" };
|
|
127
|
+
}
|
|
110
128
|
if (
|
|
111
129
|
attempt.expectedSchemaVersion !== undefined &&
|
|
112
130
|
(await schemaVersion()) !== attempt.expectedSchemaVersion
|
|
@@ -173,6 +191,36 @@ export function createMemoryStorage(options: MemoryStorageOptions = {}): MemoryS
|
|
|
173
191
|
return state.secretsPublicKey;
|
|
174
192
|
},
|
|
175
193
|
|
|
194
|
+
async base() {
|
|
195
|
+
return { revision: state.baseRevision, resolvedRef: state.baseRef };
|
|
196
|
+
},
|
|
197
|
+
|
|
198
|
+
async changes(_base, snapshot) {
|
|
199
|
+
const baseFiles = await Promise.all(
|
|
200
|
+
Object.entries(state.base).map(async ([file, content]) => ({
|
|
201
|
+
file,
|
|
202
|
+
version: await version(content),
|
|
203
|
+
size: new TextEncoder().encode(content).byteLength,
|
|
204
|
+
})),
|
|
205
|
+
);
|
|
206
|
+
const baseVersion = new Map(baseFiles.map((f) => [f.file, f.version]));
|
|
207
|
+
const current = new Map(snapshot.files.map((f) => [f.file, f.version]));
|
|
208
|
+
const files = new Set<string>();
|
|
209
|
+
for (const [file, v] of current) if (baseVersion.get(file) !== v) files.add(file);
|
|
210
|
+
for (const file of baseVersion.keys()) if (!current.has(file)) files.add(file);
|
|
211
|
+
return { files: [...files], baseFiles };
|
|
212
|
+
},
|
|
213
|
+
|
|
214
|
+
async readBaseFiles(_base, files) {
|
|
215
|
+
const out: Record<string, StoredFileBody> = Object.create(null);
|
|
216
|
+
for (const file of files) {
|
|
217
|
+
if (!(file in state.base)) continue;
|
|
218
|
+
const text = state.base[file]!;
|
|
219
|
+
out[file] = { text, version: await version(text) };
|
|
220
|
+
}
|
|
221
|
+
return out;
|
|
222
|
+
},
|
|
223
|
+
|
|
176
224
|
commit(attempt): Promise<CommitResult> {
|
|
177
225
|
// Serialized, so a commit's checks and its mutation are one atomic step.
|
|
178
226
|
const run = commitQueue.then(() => commitNow(attempt));
|
|
@@ -194,6 +242,15 @@ export function createMemoryStorage(options: MemoryStorageOptions = {}): MemoryS
|
|
|
194
242
|
else state.files[file] = content;
|
|
195
243
|
},
|
|
196
244
|
|
|
245
|
+
setBase(files) {
|
|
246
|
+
state.base = { ...files };
|
|
247
|
+
state.baseRevision = `base-${Math.random().toString(36).slice(2, 10)}`;
|
|
248
|
+
},
|
|
249
|
+
|
|
250
|
+
commitBase() {
|
|
251
|
+
storage.setBase(state.files);
|
|
252
|
+
},
|
|
253
|
+
|
|
197
254
|
setSchema(schema) {
|
|
198
255
|
state.schema =
|
|
199
256
|
schema === null || typeof schema === "string" ? schema : JSON.stringify(schema);
|
package/src/protocol/storage.ts
CHANGED
|
@@ -2,7 +2,9 @@
|
|
|
2
2
|
* The storage interface a content-protocol backend implements.
|
|
3
3
|
*
|
|
4
4
|
* A storage is deliberately small: a snapshot of file names and versions,
|
|
5
|
-
* reading file bodies, reading the schema,
|
|
5
|
+
* reading file bodies, reading the schema, one atomic commit attempt, and its
|
|
6
|
+
* base (what `changes.*` compares the content with: the files that differ
|
|
7
|
+
* from it, and reading base bodies).
|
|
6
8
|
* The protocol core (`createContentHandler`) owns everything else: the
|
|
7
9
|
* JSON-RPC layer, validation, the file-name rule, the secret guard, limits and
|
|
8
10
|
* retries. Storages deal in *files* inside `<root>/.deco/blocks`; entry names
|
|
@@ -84,6 +86,41 @@ export interface CommitAttempt {
|
|
|
84
86
|
* commit time differs, checked atomically with `expected`.
|
|
85
87
|
*/
|
|
86
88
|
expectedSchemaVersion?: string | null;
|
|
89
|
+
/**
|
|
90
|
+
* The base revision (`StorageBase.revision`) the core computed the write
|
|
91
|
+
* from, when the write depends on it (`changes.discard` restores base
|
|
92
|
+
* bytes). When set, a storage MUST reject the attempt as stale if its base
|
|
93
|
+
* revision at commit time differs, checked atomically with `expected`.
|
|
94
|
+
*/
|
|
95
|
+
expectedBase?: string;
|
|
96
|
+
/** Why the core commits, for logs: `apply` (default) or `discard`. */
|
|
97
|
+
reason?: "apply" | "discard";
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/** What a storage's content is compared against (`changes.*`). */
|
|
101
|
+
export interface StorageBase {
|
|
102
|
+
/** An opaque revision of the base, compared only for equality (a commit sha). */
|
|
103
|
+
revision: string;
|
|
104
|
+
/** The branch the base was read from; `null` when there's none. */
|
|
105
|
+
resolvedRef: string | null;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/** The files of a snapshot that differ from the base. */
|
|
109
|
+
export interface StorageChanges {
|
|
110
|
+
/**
|
|
111
|
+
* Saved-block file names that differ from the base: in the snapshot and not
|
|
112
|
+
* in the base, in the base and not in the snapshot, or in both with
|
|
113
|
+
* different content. A file whose content equals the base's (the storage
|
|
114
|
+
* decides equality, such as honouring git's eol conversion) MUST NOT be
|
|
115
|
+
* listed, so a file a discard just restored is never listed again.
|
|
116
|
+
*/
|
|
117
|
+
files: string[];
|
|
118
|
+
/**
|
|
119
|
+
* The base's files, with their versions in the base: at least every file
|
|
120
|
+
* whose spelling group (`spellingKey`) a file in `files` belongs to; more
|
|
121
|
+
* is fine.
|
|
122
|
+
*/
|
|
123
|
+
baseFiles: StorageFile[];
|
|
87
124
|
}
|
|
88
125
|
|
|
89
126
|
export type CommitResult =
|
|
@@ -115,6 +152,19 @@ export interface ContentStorage {
|
|
|
115
152
|
readSecretsPublicKey(): Promise<string | null>;
|
|
116
153
|
/** One all-or-nothing commit attempt; durable when it resolves. */
|
|
117
154
|
commit(attempt: CommitAttempt): Promise<CommitResult>;
|
|
155
|
+
/**
|
|
156
|
+
* The current base, cheaply: the core calls it on every `changes.list`
|
|
157
|
+
* poll. Throws `StorageUnavailableError` when the base can't be read now.
|
|
158
|
+
*/
|
|
159
|
+
base(): Promise<StorageBase>;
|
|
160
|
+
/** The files of `snapshot` that differ from `base`, and the base files they touch. */
|
|
161
|
+
changes(base: StorageBase, snapshot: StorageSnapshot): Promise<StorageChanges>;
|
|
162
|
+
/**
|
|
163
|
+
* Reads base file bodies, as a checkout would write them (a leading BOM
|
|
164
|
+
* kept: a discard writes the text back as is), each with its version in the
|
|
165
|
+
* base. A file the base doesn't have is left out.
|
|
166
|
+
*/
|
|
167
|
+
readBaseFiles(base: StorageBase, files: string[]): Promise<Record<string, StoredFileBody>>;
|
|
118
168
|
/** Stores an uploaded file, never overwriting one; returns the stored file name. */
|
|
119
169
|
putAsset?(name: string, body: Uint8Array): Promise<{ name: string }>;
|
|
120
170
|
}
|