@volter/world-core 3.0.154 → 3.0.156
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/generated/pack-facts.json +140 -0
- package/dist/inject.cjs +150 -86
- package/dist/src/actions.d.ts +15 -0
- package/dist/src/actions.js +82 -21
- package/dist/src/ancestry.d.ts +5 -4
- package/dist/src/ancestry.js +53 -28
- package/dist/src/changeset.js +1 -1
- package/dist/src/clickhouse/compression.d.ts +28 -0
- package/dist/src/clickhouse/compression.js +281 -0
- package/dist/src/clickhouse/index.d.ts +1 -0
- package/dist/src/clickhouse/index.js +1 -0
- package/dist/src/client-bundle.js +15 -15
- package/dist/src/derived-core.d.ts +12 -1
- package/dist/src/derived-core.js +145 -11
- package/dist/src/derived.js +3 -0
- package/dist/src/fork.js +1 -1
- package/dist/src/graphql-wire.d.ts +5 -0
- package/dist/src/graphql-wire.js +6 -6
- package/dist/src/grpc-wire.js +2 -1
- package/dist/src/h2.js +63 -10
- package/dist/src/head.d.ts +3 -0
- package/dist/src/head.js +9 -0
- package/dist/src/history.d.ts +1 -1
- package/dist/src/history.js +7 -7
- package/dist/src/index.d.ts +4 -1
- package/dist/src/index.js +5 -1
- package/dist/src/log.d.ts +23 -3
- package/dist/src/log.js +160 -37
- package/dist/src/managed-original-inputs.d.ts +101 -0
- package/dist/src/managed-original-inputs.js +125 -0
- package/dist/src/mongodb/client.d.ts +8 -0
- package/dist/src/mongodb/client.js +27 -0
- package/dist/src/mongodb/index.d.ts +2 -0
- package/dist/src/mongodb/index.js +5 -0
- package/dist/src/mongodb/stream.d.ts +74 -0
- package/dist/src/mongodb/stream.js +206 -0
- package/dist/src/observe.js +1 -1
- package/dist/src/pack-fetch.d.ts +2 -1
- package/dist/src/pack-fetch.js +21 -9
- package/dist/src/packRegistry.d.ts +10 -14
- package/dist/src/packRegistry.js +2 -13
- package/dist/src/protocol.d.ts +13 -0
- package/dist/src/protocol.js +13 -0
- package/dist/src/redis/engine.d.ts +10 -4
- package/dist/src/redis/engine.js +103 -32
- package/dist/src/redis/member-storage.js +20 -8
- package/dist/src/redis/service-times.d.ts +25 -0
- package/dist/src/redis/service-times.js +87 -0
- package/dist/src/redis/stream.d.ts +2 -1
- package/dist/src/redis/stream.js +109 -20
- package/dist/src/runtime-state.d.ts +11 -0
- package/dist/src/runtime-state.js +13 -0
- package/dist/src/runtime.d.ts +4 -2
- package/dist/src/runtime.js +5 -2
- package/dist/src/serve-http.d.ts +2 -0
- package/dist/src/serve-http.js +21 -4
- package/dist/src/serve.d.ts +10 -0
- package/dist/src/serve.js +43 -3
- package/dist/src/storage.d.ts +2 -1
- package/dist/src/storage.js +6 -3
- package/dist/src/world-boot.d.ts +1 -0
- package/dist/src/world-boot.js +6 -0
- package/dist/src/world-clock.js +6 -2
- package/dist/src/world-store.d.ts +108 -7
- package/dist/src/world-store.js +883 -103
- package/dist/src/xml.d.ts +30 -0
- package/dist/src/xml.js +74 -0
- package/generated/pack-facts.json +140 -0
- package/inject.cjs +150 -86
- package/package.json +20 -1
- package/src/actions.ts +78 -21
- package/src/ancestry.ts +46 -24
- package/src/changeset.ts +1 -1
- package/src/clickhouse/compression.ts +172 -0
- package/src/clickhouse/index.ts +2 -0
- package/src/client-bundle.ts +13 -14
- package/src/derived-core.ts +142 -11
- package/src/derived.ts +3 -0
- package/src/fork.ts +1 -1
- package/src/graphql-wire.ts +9 -6
- package/src/grpc-wire.ts +2 -1
- package/src/h2.ts +46 -5
- package/src/head.ts +9 -0
- package/src/history.ts +7 -7
- package/src/index.ts +6 -1
- package/src/log.ts +147 -32
- package/src/managed-original-inputs.ts +163 -0
- package/src/mongodb/client.ts +27 -0
- package/src/mongodb/index.ts +5 -0
- package/src/mongodb/stream.ts +213 -0
- package/src/observe.ts +1 -1
- package/src/pack-fetch.ts +23 -11
- package/src/packRegistry.ts +3 -15
- package/src/protocol.ts +13 -0
- package/src/redis/engine.ts +100 -57
- package/src/redis/member-storage.ts +17 -9
- package/src/redis/service-times.ts +81 -0
- package/src/redis/stream.ts +66 -14
- package/src/runtime-state.ts +14 -0
- package/src/runtime.ts +6 -1
- package/src/serve-http.ts +23 -4
- package/src/serve.ts +45 -3
- package/src/storage.ts +5 -3
- package/src/world-boot.ts +6 -0
- package/src/world-clock.ts +6 -2
- package/src/world-store.ts +704 -91
- package/src/xml.ts +71 -0
package/dist/src/redis/stream.js
CHANGED
|
@@ -1,15 +1,38 @@
|
|
|
1
1
|
import { RedisStatus, commandInteger, commandShapeError, globMatch } from "./engine.js";
|
|
2
2
|
import { binaryBytes, binaryText, inlineCommand, readResp, respReply } from "./frames.js";
|
|
3
|
-
|
|
3
|
+
import { worldNow } from "../world-clock.js";
|
|
4
|
+
import { redisServiceTimes as times } from "./service-times.js";
|
|
5
|
+
export function createRedisStream(given) {
|
|
6
|
+
// The pack's run, its time counted for the command it serves (service-times.ts).
|
|
7
|
+
const options = { ...given, run: async (commands, database) => { const began = times.now(); times.lapFrom(); try {
|
|
8
|
+
const ran = await given.run(commands, database);
|
|
9
|
+
if (ran.wrote)
|
|
10
|
+
times.wrote();
|
|
11
|
+
return ran;
|
|
12
|
+
}
|
|
13
|
+
finally {
|
|
14
|
+
times.lapEnd('8 answer read by the stream');
|
|
15
|
+
times.add('run', times.now() - began);
|
|
16
|
+
} } };
|
|
4
17
|
const sessions = new Set();
|
|
5
|
-
let nextId = 0, commandCount = 0;
|
|
18
|
+
let nextId = 0, commandCount = 0, waiting = 0;
|
|
6
19
|
let serial = Promise.resolve();
|
|
7
20
|
const ordered = (f) => { const p = serial.then(f); serial = p.catch(() => { }); return p; };
|
|
8
21
|
const error = (message) => ({ error: message });
|
|
9
22
|
const ok = new RedisStatus('OK');
|
|
10
|
-
const
|
|
11
|
-
|
|
12
|
-
|
|
23
|
+
const connectionTrace = (s, stage, command) => {
|
|
24
|
+
try {
|
|
25
|
+
console.error('[redis-twin-connection]', JSON.stringify({ at: performance.timeOrigin + times.now(), pid: globalThis.process?.pid ?? 0,
|
|
26
|
+
session: s.id, stage, received: s.received, answered: s.answered, ...(command ? { command } : {}) }));
|
|
27
|
+
}
|
|
28
|
+
catch { /* diagnostic output cannot change a connection */ }
|
|
29
|
+
};
|
|
30
|
+
const answer = (s, v) => { if (!s.closed) {
|
|
31
|
+
times.timed('reply', () => s.sink.write(binaryBytes(respReply(v))));
|
|
32
|
+
s.answered += 1;
|
|
33
|
+
} };
|
|
34
|
+
const close = (s) => { if (!s.closed)
|
|
35
|
+
connectionTrace(s, 'closed'); s.closed = true; if (s.blocked?.timer)
|
|
13
36
|
clearTimeout(s.blocked.timer); sessions.delete(s); };
|
|
14
37
|
const unblock = (s, v) => { if (s.blocked?.timer)
|
|
15
38
|
clearTimeout(s.blocked.timer); s.blocked = undefined; answer(s, v); queueMicrotask(() => { void s.pump?.(); }); };
|
|
@@ -26,12 +49,26 @@ export function createRedisStream(options) {
|
|
|
26
49
|
}
|
|
27
50
|
throw new Error('the blocking command has no executor');
|
|
28
51
|
};
|
|
29
|
-
|
|
30
|
-
|
|
52
|
+
// A CLIENT BLOCKED ON KEYS IS ASKED AGAIN ONLY WHEN ONE OF ITS KEYS WAS WRITTEN. Every write asked every blocked
|
|
53
|
+
// client again, each ask a whole run for each of its keys: Twenty's queue workers hold a blocked client apiece, and
|
|
54
|
+
// its twin made 59 to 89 runs for 31 to 41 commands in a two-second window, each run about 20 ms, with every
|
|
55
|
+
// other command waiting behind them (Twenty 51: commands waited 0.6 to 5.9 s). `touched` is the keys the run
|
|
56
|
+
// wrote, by name, whatever its database (a name written in another database asks a client that then finds
|
|
57
|
+
// nothing, never the reverse). Without it (an EXEC's queue, a run that does not say, a command that writes
|
|
58
|
+
// another database's keys) every blocked client is asked, as before.
|
|
59
|
+
const wake = async (touched) => {
|
|
60
|
+
for (const s of sessions) {
|
|
61
|
+
if (!s.blocked)
|
|
62
|
+
continue;
|
|
63
|
+
if (touched && !s.blocked.command.slice(1, -1).some((key) => touched.includes(key)))
|
|
64
|
+
continue;
|
|
31
65
|
const v = await blockRun(s, s.blocked.command);
|
|
32
66
|
if (v !== null)
|
|
33
67
|
unblock(s, v);
|
|
34
|
-
}
|
|
68
|
+
}
|
|
69
|
+
};
|
|
70
|
+
/** Commands that write keys of a database other than their session's: their run's `touched` does not name them all. */
|
|
71
|
+
const CROSS_DATABASE = new Set(['MOVE', 'COPY', 'SWAPDB']);
|
|
35
72
|
const command = async (s, argv, executing = false) => {
|
|
36
73
|
if (!argv.length || s.closed)
|
|
37
74
|
return;
|
|
@@ -272,12 +309,32 @@ export function createRedisStream(options) {
|
|
|
272
309
|
}
|
|
273
310
|
const initial = await options.run([], s.database);
|
|
274
311
|
s.blocked = { command: argv, deadline: timeout === 0 ? Infinity : (initial.instant ?? 0) + timeout };
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
312
|
+
// A BLOCKED CLIENT WAITS FOR A WRITE OR ITS DEADLINE, AND ASKS THE TREE NOTHING MEANWHILE. A write through this
|
|
313
|
+
// twin wakes it (wake, after every command that wrote). Its deadline is in the World's time, so the timer
|
|
314
|
+
// reads the World's clock (world-clock.ts worldNow: one check of the clock file, kept while it stands) and
|
|
315
|
+
// nothing else. It ran a whole empty request every 50 ms to read that time: the pack's fetch, its context
|
|
316
|
+
// and the keyspace's image with its tree stamp, for every blocked client, the whole time it was blocked.
|
|
317
|
+
// Twenty's queue workers each hold one blocked client: its Redis twin made about 18,700 stamps in a run, 56,145
|
|
318
|
+
// stat requests, and spent 55 s of 114 busy in them, with no command to answer. No timeout is no timer; a
|
|
319
|
+
// deadline far off is checked less often, never more than a second apart, and to 50 ms as it nears.
|
|
320
|
+
const poll = () => {
|
|
321
|
+
if (!s.blocked || s.closed || s.blocked.deadline === Infinity)
|
|
322
|
+
return;
|
|
323
|
+
let left;
|
|
324
|
+
try {
|
|
325
|
+
left = s.blocked.deadline - Date.parse(worldNow());
|
|
326
|
+
}
|
|
327
|
+
catch (cause) {
|
|
328
|
+
unblock(s, error(`ERR twin: ${cause instanceof Error ? cause.message : String(cause)}`.replace(/[\r\n]+/g, ' ')));
|
|
329
|
+
return;
|
|
330
|
+
}
|
|
331
|
+
if (left <= 0) {
|
|
332
|
+
void ordered(async () => { if (s.blocked)
|
|
333
|
+
unblock(s, null); });
|
|
334
|
+
return;
|
|
335
|
+
}
|
|
336
|
+
s.blocked.timer = setTimeout(poll, Math.max(50, Math.min(1000, left / 2)));
|
|
337
|
+
};
|
|
281
338
|
poll();
|
|
282
339
|
return;
|
|
283
340
|
}
|
|
@@ -286,17 +343,45 @@ export function createRedisStream(options) {
|
|
|
286
343
|
const item = res.items[0];
|
|
287
344
|
answer(s, 'error' in item ? item : item.result);
|
|
288
345
|
if (res.wrote && !executing)
|
|
289
|
-
await wake();
|
|
346
|
+
await wake(CROSS_DATABASE.has(cmd) ? undefined : res.touched);
|
|
290
347
|
}
|
|
291
348
|
}
|
|
292
349
|
};
|
|
293
350
|
return (sink) => {
|
|
294
|
-
const s = { id: ++nextId, name: '', lib: {}, database: '0', authenticated: !options.password, queue: null, dirty: false, subscriptions: new Set(), patterns: new Set(), sink, closed: false };
|
|
351
|
+
const s = { id: ++nextId, name: '', lib: {}, database: '0', authenticated: !options.password, queue: null, dirty: false, subscriptions: new Set(), patterns: new Set(), sink, closed: false, received: 0, answered: 0 };
|
|
295
352
|
sessions.add(s);
|
|
353
|
+
connectionTrace(s, 'opened');
|
|
296
354
|
let buffer = '', chain = Promise.resolve();
|
|
297
355
|
const pending = [];
|
|
298
|
-
|
|
299
|
-
|
|
356
|
+
// A command whose storage fails (a World storage lock that timed out, a write that could not be made) answers its
|
|
357
|
+
// client with an error and the session goes on, as Redis answers one failed command: it was an unhandled
|
|
358
|
+
// rejection, and the twin's process exited with every client's connection (Twenty 46).
|
|
359
|
+
const pump = async () => {
|
|
360
|
+
while (pending.length && !s.blocked && !s.closed) {
|
|
361
|
+
const { argv, received } = pending.shift();
|
|
362
|
+
try {
|
|
363
|
+
await ordered(async () => {
|
|
364
|
+
waiting -= 1;
|
|
365
|
+
times.begin(received, waiting, { session: s.id, command: argv[0]?.toUpperCase().slice(0, 32) ?? '' });
|
|
366
|
+
let threw = false;
|
|
367
|
+
try {
|
|
368
|
+
await command(s, argv);
|
|
369
|
+
}
|
|
370
|
+
catch (cause) {
|
|
371
|
+
threw = true;
|
|
372
|
+
throw cause;
|
|
373
|
+
}
|
|
374
|
+
finally {
|
|
375
|
+
times.end(threw);
|
|
376
|
+
}
|
|
377
|
+
});
|
|
378
|
+
}
|
|
379
|
+
catch (cause) {
|
|
380
|
+
if (!s.closed)
|
|
381
|
+
answer(s, error(`ERR twin: ${cause instanceof Error ? cause.message : String(cause)}`.replace(/[\r\n]+/g, ' ')));
|
|
382
|
+
}
|
|
383
|
+
}
|
|
384
|
+
};
|
|
300
385
|
s.pump = pump;
|
|
301
386
|
return {
|
|
302
387
|
data(chunk) { if (s.closed)
|
|
@@ -321,7 +406,11 @@ export function createRedisStream(options) {
|
|
|
321
406
|
end = at + 2;
|
|
322
407
|
}
|
|
323
408
|
buffer = buffer.slice(end);
|
|
324
|
-
pending.push(argv);
|
|
409
|
+
pending.push({ argv, received: times.now() });
|
|
410
|
+
waiting += 1;
|
|
411
|
+
s.received += 1;
|
|
412
|
+
if (s.received === 1)
|
|
413
|
+
connectionTrace(s, 'first-command', argv[0]?.toUpperCase().slice(0, 32));
|
|
325
414
|
}
|
|
326
415
|
chain = chain.then(pump);
|
|
327
416
|
}
|
|
@@ -331,7 +420,7 @@ export function createRedisStream(options) {
|
|
|
331
420
|
sink.end();
|
|
332
421
|
} },
|
|
333
422
|
settled: () => chain,
|
|
334
|
-
close: () => close(s),
|
|
423
|
+
close: () => { waiting -= pending.length; pending.length = 0; close(s); },
|
|
335
424
|
};
|
|
336
425
|
};
|
|
337
426
|
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export { getActiveWorldStore, removeCarriedLocks } from './world-store.js';
|
|
2
|
+
export type { WorldStore } from './world-store.js';
|
|
3
|
+
export { stateDirName, withFileLock, worldPaths, worldStateRoot, WORLD_DATA_ENV } from './storage.js';
|
|
4
|
+
export { assertStateRemovable, withAncestryLock, withStateRemoval } from './ancestry.js';
|
|
5
|
+
export { readBranchMeta } from './log.js';
|
|
6
|
+
export { volterHome } from './volter-home.js';
|
|
7
|
+
export { installedPackageDir, installedVendorPacks, isDerivedPack, packRepository, vendorOfPackModule } from './pack-paths.js';
|
|
8
|
+
export { PROTOCOL_VERSION, PROTOCOL_MAJOR, protocolStanding } from './protocol.js';
|
|
9
|
+
export { WORLD_BOOT_PATH } from './serve-http.js';
|
|
10
|
+
export { WORLD_ENV_NAMES_ENV } from './world-env.js';
|
|
11
|
+
export { beginPlaceholderPull, endPlaceholderPull, placeholderPullActive } from './placeholder-remote.js';
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
// The lifecycle's state/identity primitives. Re-export their canonical modules
|
|
2
|
+
// so the active store, locks and scopes are shared with the full runtime and packs.
|
|
3
|
+
// Booting an empty World does not require pack registration or derived engines.
|
|
4
|
+
export { getActiveWorldStore, removeCarriedLocks } from "./world-store.js";
|
|
5
|
+
export { stateDirName, withFileLock, worldPaths, worldStateRoot, WORLD_DATA_ENV } from "./storage.js";
|
|
6
|
+
export { assertStateRemovable, withAncestryLock, withStateRemoval } from "./ancestry.js";
|
|
7
|
+
export { readBranchMeta } from "./log.js";
|
|
8
|
+
export { volterHome } from "./volter-home.js";
|
|
9
|
+
export { installedPackageDir, installedVendorPacks, isDerivedPack, packRepository, vendorOfPackModule } from "./pack-paths.js";
|
|
10
|
+
export { PROTOCOL_VERSION, PROTOCOL_MAJOR, protocolStanding } from "./protocol.js";
|
|
11
|
+
export { WORLD_BOOT_PATH } from "./serve-http.js";
|
|
12
|
+
export { WORLD_ENV_NAMES_ENV } from "./world-env.js";
|
|
13
|
+
export { beginPlaceholderPull, endPlaceholderPull, placeholderPullActive } from "./placeholder-remote.js";
|
package/dist/src/runtime.d.ts
CHANGED
|
@@ -38,7 +38,7 @@ export { SqlWorldStore } from './world-store-sql.js';
|
|
|
38
38
|
export type { SqlExec } from './world-store-sql.js';
|
|
39
39
|
export { FsBlobStore, MemoryBlobStore, blobDigest, getActiveBlobStore, setActiveBlobStore, withBlobStore, readBlobRange, } from './blob-store.js';
|
|
40
40
|
export type { BlobStore } from './blob-store.js';
|
|
41
|
-
export { applyTwinWrite, applyTwinWriteAtomic, createTwinServer, CALLER_HEADER, callerForHeader, callerFromHeader, journalTwinRequest, readTwinRequestJournalFrom, type TwinRequestJournalCursor, twinRequestJournalFailures, twinRequestCredentials, readTwinRequestJournal, recordedInput, resolveTwinRead, twinRequestJournalEnabled, twinRequestJournalPath, twinResources, } from './serve.js';
|
|
41
|
+
export { applyTwinWrite, applyTwinWrites, applyTwinWriteAtomic, createTwinServer, CALLER_HEADER, callerForHeader, callerFromHeader, journalTwinRequest, readTwinRequestJournalFrom, type TwinRequestJournalCursor, twinRequestJournalFailures, twinRequestCredentials, readTwinRequestJournal, recordedInput, resolveTwinRead, twinRequestJournalEnabled, twinRequestJournalPath, twinResources, } from './serve.js';
|
|
42
42
|
export type { AtomicTwinWriteDecision, TwinRequestJournalEntry, TwinResource, TwinWriteInput, TwinWriteResult, } from './serve.js';
|
|
43
43
|
export { createTwinProxy } from './proxy.js';
|
|
44
44
|
export type { TwinProxy, TwinProxyOptions, VendorRoute } from './proxy.js';
|
|
@@ -52,6 +52,7 @@ export { getActivePackAssets, setActivePackAssets, packAsset, assetContentType,
|
|
|
52
52
|
export { installedPackageDir, installedVendorPacks, packVendor, packRepository, isDerivedPack, vendorPackageDir, vendorOfPackModule } from './pack-paths.js';
|
|
53
53
|
export * as git from './git/index.js';
|
|
54
54
|
export * as s3 from './s3/wire.js';
|
|
55
|
+
export * as xml from './xml.js';
|
|
55
56
|
export { PLACEHOLDER_REMOTE, beginPlaceholderPull, endPlaceholderPull, placeholderPullActive, placeholderPullMarkerPath, withPlaceholderPull, placeholderEventsFor } from './placeholder-remote.js';
|
|
56
57
|
export { aliasesFrom, ownFields, subjectHistory, rebaseBranch, cutCheckpoint, appendParentEntry, wholeLog, branchEntries, branchLogPath, branchMetaPath, CHECKPOINT_EVERY, dropCheckpoint, foldEntries, landedCopy, landedIds, parentEntries, parentLogPath, position, readBranchMeta, readTree, toEntry, toEvent, unpushedEntries, writeBranchMeta } from './log.js';
|
|
57
58
|
export type { BranchMeta, Entry, EntryOp, Receipt } from './log.js';
|
|
@@ -69,7 +70,7 @@ export { observeResource, observeResources, collectObservations, foldObservation
|
|
|
69
70
|
export type { ObservedResource, ObserveReport, Observation } from './observe.js';
|
|
70
71
|
export { readTreeMap, readParentTreeMap, treeStamp, treeChangesSince, originEntries, appendOriginEntry, originLogPath, isUrlParent, parentPosition, positionAt, splitsBatch, assertBatchBoundary } from './log.js';
|
|
71
72
|
export { referenceField, registerReferences, packReferences, resolveReferences, type ReferenceDeclaration } from './references.js';
|
|
72
|
-
export { serveHttp, serveStream, connectWebSocket, nodeBuiltin, WORLD_BOOT_PATH, type HttpServer, type ServeHttpOptions, type HttpHandler } from './serve-http.js';
|
|
73
|
+
export { serveHttp, serveStream, connectWebSocket, nodeBuiltin, servedHandlerOf, WORLD_BOOT_PATH, type HttpServer, type ServeHttpOptions, type HttpHandler } from './serve-http.js';
|
|
73
74
|
export { fileResponse, contentTypeOf } from './file-response.js';
|
|
74
75
|
export { bundleClient, filePathOf } from './client-bundle.js';
|
|
75
76
|
export { brandTokensResponse } from './brand-tokens.js';
|
|
@@ -86,3 +87,4 @@ export { foldHistory } from './log.js';
|
|
|
86
87
|
export { volterHome } from './volter-home.js';
|
|
87
88
|
export type { MachinePool, MachineSpec, MachineProcess, MachineStatus, MachineFiles, MachineFileStat, GuestImage, LightPoolOptions } from './machines.js';
|
|
88
89
|
export { boardFrameKey } from './board-keys.js';
|
|
90
|
+
export * as managedOriginalInputs from './managed-original-inputs.js';
|
package/dist/src/runtime.js
CHANGED
|
@@ -55,7 +55,7 @@ export { SqlWorldStore } from "./world-store-sql.js";
|
|
|
55
55
|
// The blob seam (runtime contract R11): byte storage behind byte-carrying handlers, so a
|
|
56
56
|
// serverless namespace puts bytes in object storage while local worlds keep today's layout.
|
|
57
57
|
export { FsBlobStore, MemoryBlobStore, blobDigest, getActiveBlobStore, setActiveBlobStore, withBlobStore, readBlobRange, } from "./blob-store.js";
|
|
58
|
-
export { applyTwinWrite, applyTwinWriteAtomic, createTwinServer, CALLER_HEADER, callerForHeader, callerFromHeader, journalTwinRequest, readTwinRequestJournalFrom, twinRequestJournalFailures, twinRequestCredentials, readTwinRequestJournal, recordedInput, resolveTwinRead, twinRequestJournalEnabled, twinRequestJournalPath, twinResources, } from "./serve.js";
|
|
58
|
+
export { applyTwinWrite, applyTwinWrites, applyTwinWriteAtomic, createTwinServer, CALLER_HEADER, callerForHeader, callerFromHeader, journalTwinRequest, readTwinRequestJournalFrom, twinRequestJournalFailures, twinRequestCredentials, readTwinRequestJournal, recordedInput, resolveTwinRead, twinRequestJournalEnabled, twinRequestJournalPath, twinResources, } from "./serve.js";
|
|
59
59
|
export { createTwinProxy } from "./proxy.js";
|
|
60
60
|
export { forkTwin, isFork, readForkMeta, } from "./fork.js";
|
|
61
61
|
export { appendAction, appendTransactionCommit, checkPrecondition, confirmAction, listActions, listTransactionCommits, pendingActions, pushablePendingActions, isTwinBookkeeping, pendingTransactionCommits, projectResources, projectOwnerResources, OwnerStoreAmbiguousError, revertAction, TwinActionPreconditionError, subjectAliases, resolveSubjectId, runWithCorrelationId, currentCorrelationId } from "./actions.js";
|
|
@@ -73,6 +73,8 @@ export { installedPackageDir, installedVendorPacks, packVendor, packRepository,
|
|
|
73
73
|
export * as git from "./git/index.js";
|
|
74
74
|
// S3's wire (contract "S3's wire is a kernel library too"): its XML, CORS, lifecycle, delete batches, ETags, body bytes.
|
|
75
75
|
export * as s3 from "./s3/wire.js";
|
|
76
|
+
// Generic strict XML grammar; vendor interpretation remains in its consumer.
|
|
77
|
+
export * as xml from "./xml.js";
|
|
76
78
|
// The push arm: the kernel's transaction over a pack's one `perform<Name>Action` (contract section of that name).
|
|
77
79
|
// The placeholder remote (contract section of that name): the seed is a pull, never a pending write.
|
|
78
80
|
export { PLACEHOLDER_REMOTE, beginPlaceholderPull, endPlaceholderPull, placeholderPullActive, placeholderPullMarkerPath, withPlaceholderPull, placeholderEventsFor } from "./placeholder-remote.js";
|
|
@@ -88,7 +90,7 @@ export { observeResource, observeResources, collectObservations, foldObservation
|
|
|
88
90
|
export { readTreeMap, readParentTreeMap, treeStamp, treeChangesSince, originEntries, appendOriginEntry, originLogPath, isUrlParent, parentPosition, positionAt, splitsBatch, assertBatchBoundary } from "./log.js";
|
|
89
91
|
export { referenceField, registerReferences, packReferences, resolveReferences } from "./references.js";
|
|
90
92
|
// the HTTP server seam: the one place the serve path meets the runtime (Bun or Node)
|
|
91
|
-
export { serveHttp, serveStream, connectWebSocket, nodeBuiltin, WORLD_BOOT_PATH } from "./serve-http.js";
|
|
93
|
+
export { serveHttp, serveStream, connectWebSocket, nodeBuiltin, servedHandlerOf, WORLD_BOOT_PATH } from "./serve-http.js";
|
|
92
94
|
// two runtime-neutral helpers for a serve path: a file as a Response, a page's client bundle
|
|
93
95
|
export { fileResponse, contentTypeOf } from "./file-response.js";
|
|
94
96
|
export { bundleClient, filePathOf } from "./client-bundle.js";
|
|
@@ -104,3 +106,4 @@ export { captureHistory, captureParentHistory, historyChanges, historyAtInstant,
|
|
|
104
106
|
export { foldHistory } from "./log.js";
|
|
105
107
|
export { volterHome } from "./volter-home.js";
|
|
106
108
|
export { boardFrameKey } from "./board-keys.js";
|
|
109
|
+
export * as managedOriginalInputs from "./managed-original-inputs.js";
|
package/dist/src/serve-http.d.ts
CHANGED
|
@@ -45,6 +45,8 @@ export type HttpServer = {
|
|
|
45
45
|
export declare function setServeDecorator(fn: ((options: ServeHttpOptions) => HttpHandler) | undefined): void;
|
|
46
46
|
/** The path a World asks a service it booted, when that service's port answers, which boot it belongs to. */
|
|
47
47
|
export declare const WORLD_BOOT_PATH = "/__volter/world-boot";
|
|
48
|
+
/** The handler serving `port` in this process, when a server made through the seam listens there. */
|
|
49
|
+
export declare function servedHandlerOf(port: number): HttpHandler | undefined;
|
|
48
50
|
export declare function serveHttp(options: ServeHttpOptions): Promise<HttpServer>;
|
|
49
51
|
/** A Node builtin without a static import — `process.getBuiltinModule` (Node 22.3+, Bun): what a
|
|
50
52
|
* module that also rides into a browser bundle reaches `node:crypto` with. A bare `require()`
|
package/dist/src/serve-http.js
CHANGED
|
@@ -4,9 +4,10 @@
|
|
|
4
4
|
// as a standard Request and the handler's Response written back. Async, because binding a port is
|
|
5
5
|
// asynchronous on Node — Bun's synchronous bind is the special case.
|
|
6
6
|
//
|
|
7
|
-
// Nothing here runs at module scope, and `node:http` is reached through `process.getBuiltinModule`
|
|
7
|
+
// Nothing here runs at module scope (the served-handler registry is made on first use), and `node:http` is reached through `process.getBuiltinModule`
|
|
8
8
|
// rather than a static import: this module rides into browser client bundles through the kernel's
|
|
9
9
|
// entrypoint, where a static `node:` import is the 2026-09-06 class of break.
|
|
10
|
+
import { worldBootAnswer } from "./world-boot.js";
|
|
10
11
|
/** One socket library at the host seam, shared by inbound upgrades and authenticated outbound relays. */
|
|
11
12
|
const socketLibrary = () => import('ws');
|
|
12
13
|
export async function connectWebSocket(url, headers, protocols = []) {
|
|
@@ -19,6 +20,22 @@ export function setServeDecorator(fn) { decorate = fn; }
|
|
|
19
20
|
const bunRuntime = () => { const b = globalThis.Bun; return b && typeof b.serve === 'function' ? b : undefined; };
|
|
20
21
|
/** The path a World asks a service it booted, when that service's port answers, which boot it belongs to. */
|
|
21
22
|
export const WORLD_BOOT_PATH = '/__volter/world-boot';
|
|
23
|
+
/** Each listening server's own handler, by its port, as the seam serves it (the boot door answered, nothing journaled):
|
|
24
|
+
* the World's co-located host warms a twin in-process through it (world-runtime host.ts warmInProcess), so its warm-up is
|
|
25
|
+
* never a request on the wire nor in any journal. A server leaves it when it stops. Kept on globalThis under a registered
|
|
26
|
+
* symbol, as the reference registry is (references.ts), so a pack that resolved its own copy of this module shares it. */
|
|
27
|
+
const servedHandlers = () => (globalThis[Symbol.for('volter.servedHandlers')] ??= new Map());
|
|
28
|
+
/** The handler serving `port` in this process, when a server made through the seam listens there. */
|
|
29
|
+
export function servedHandlerOf(port) { return servedHandlers().get(port); }
|
|
30
|
+
/** A server's handler kept by its port until the server stops. */
|
|
31
|
+
function keepHandler(server, handler) {
|
|
32
|
+
const port = server.port;
|
|
33
|
+
if (!port)
|
|
34
|
+
return server;
|
|
35
|
+
servedHandlers().set(port, handler);
|
|
36
|
+
return { ...server, stop: async (closeActive) => { if (servedHandlers().get(port) === handler)
|
|
37
|
+
servedHandlers().delete(port); await server.stop(closeActive); } };
|
|
38
|
+
}
|
|
22
39
|
export async function serveHttp(options) {
|
|
23
40
|
// Inside a World every server made through the seam answers which boot it belongs to, so the World can tell its own
|
|
24
41
|
// twin from a leftover World's that holds the same port. Outside a World nothing changes.
|
|
@@ -33,7 +50,7 @@ export async function serveHttp(options) {
|
|
|
33
50
|
const worldBoot = globalThis.process?.env?.VOLTER_WORLD_BOOT_ID;
|
|
34
51
|
if (worldBoot) {
|
|
35
52
|
const served = options.fetch;
|
|
36
|
-
options = { ...options, fetch: (request) => (new URL(request.url).pathname === WORLD_BOOT_PATH ?
|
|
53
|
+
options = { ...options, fetch: (request) => (new URL(request.url).pathname === WORLD_BOOT_PATH ? worldBootAnswer(worldBoot) : served(request)) };
|
|
37
54
|
}
|
|
38
55
|
// loopback by default: an unnamed interface is never every interface
|
|
39
56
|
const hostname = options.hostname ?? '127.0.0.1';
|
|
@@ -60,9 +77,9 @@ export async function serveHttp(options) {
|
|
|
60
77
|
} : {}),
|
|
61
78
|
});
|
|
62
79
|
const port = server.port ?? 0;
|
|
63
|
-
return { hostname, port, url: server.url instanceof URL ? server.url : new URL(`http://${urlHost}:${port}/`), stop: async (closeActive = true) => { await server.stop(closeActive); } };
|
|
80
|
+
return keepHandler({ hostname, port, url: server.url instanceof URL ? server.url : new URL(`http://${urlHost}:${port}/`), stop: async (closeActive = true) => { await server.stop(closeActive); } }, options.fetch);
|
|
64
81
|
}
|
|
65
|
-
return serveOnNode({ ...options, hostname, urlHost, fetch: decorate ? decorate(options) : options.fetch });
|
|
82
|
+
return keepHandler(await serveOnNode({ ...options, hostname, urlHost, fetch: decorate ? decorate(options) : options.fetch }), options.fetch);
|
|
66
83
|
}
|
|
67
84
|
/** A Node builtin without a static import — `process.getBuiltinModule` (Node 22.3+, Bun): what a
|
|
68
85
|
* module that also rides into a browser bundle reaches `node:crypto` with. A bare `require()`
|
package/dist/src/serve.d.ts
CHANGED
|
@@ -184,6 +184,16 @@ export declare function applyTwinWrite(service: string, write: TwinWriteInput, r
|
|
|
184
184
|
result: TwinWriteResult;
|
|
185
185
|
resource: TwinResource;
|
|
186
186
|
}>;
|
|
187
|
+
/**
|
|
188
|
+
* Several local writes of one service, landed as `applyTwinWrite` lands each called in turn — the same actions, ids and
|
|
189
|
+
* order — under one lock and one append (actions.ts `appendActionOccurrences`). Writes that need the one-at-a-time
|
|
190
|
+
* path take it: a caller's identity (`idempotencyKey`/`actionId`), preconditions, or a head that performs each entry as
|
|
191
|
+
* it lands. An input is offloaded once per distinct value, as each write's own offload would name it.
|
|
192
|
+
*/
|
|
193
|
+
export declare function applyTwinWrites(service: string, writes: readonly TwinWriteInput[], root?: string): Promise<Array<{
|
|
194
|
+
result: TwinWriteResult;
|
|
195
|
+
resource: TwinResource;
|
|
196
|
+
}>>;
|
|
187
197
|
export declare function createTwinServer(options: {
|
|
188
198
|
service: string;
|
|
189
199
|
root?: string;
|
package/dist/src/serve.js
CHANGED
|
@@ -12,12 +12,13 @@ import { parseExactJson } from "./exact-json.js";
|
|
|
12
12
|
// reality). Forking is a separate data operation (fork.ts), never a serve mode.
|
|
13
13
|
import { AsyncLocalStorage } from 'node:async_hooks';
|
|
14
14
|
import { serveHttp, setServeDecorator, WORLD_BOOT_PATH } from "./serve-http.js";
|
|
15
|
-
import { performAtHead } from "./head.js";
|
|
15
|
+
import { performAtHead, performsAtHeadNow } from "./head.js";
|
|
16
16
|
import { withHeaders } from "./with-headers.js";
|
|
17
17
|
import { createHash } from 'node:crypto';
|
|
18
18
|
import { dirname, join } from 'node:path';
|
|
19
19
|
import { hashFieldValue } from "./hash.js";
|
|
20
|
-
import { appendActionIfAbsent, appendActionOccurrence, decideAndAppendAction, projectResources } from "./actions.js";
|
|
20
|
+
import { appendActionIfAbsent, appendActionOccurrence, appendActionOccurrences, decideAndAppendAction, projectResources } from "./actions.js";
|
|
21
|
+
import { readSubject } from "./log.js";
|
|
21
22
|
import { observeWorldPaths, worldPaths } from "./storage.js";
|
|
22
23
|
import { getActiveWorldStore } from "./world-store.js";
|
|
23
24
|
import { getActiveBlobStore } from "./blob-store.js";
|
|
@@ -596,11 +597,50 @@ export async function applyTwinWrite(service, write, root) {
|
|
|
596
597
|
return {
|
|
597
598
|
result,
|
|
598
599
|
get resource() {
|
|
599
|
-
return resolved ??=
|
|
600
|
+
return resolved ??= readSubject(service, write.subjectType, answerId, root)
|
|
600
601
|
?? { id: answerId, type: write.subjectType, updatedAt: landed.occurredAt, ...write.fields };
|
|
601
602
|
},
|
|
602
603
|
};
|
|
603
604
|
}
|
|
605
|
+
/**
|
|
606
|
+
* Several local writes of one service, landed as `applyTwinWrite` lands each called in turn — the same actions, ids and
|
|
607
|
+
* order — under one lock and one append (actions.ts `appendActionOccurrences`). Writes that need the one-at-a-time
|
|
608
|
+
* path take it: a caller's identity (`idempotencyKey`/`actionId`), preconditions, or a head that performs each entry as
|
|
609
|
+
* it lands. An input is offloaded once per distinct value, as each write's own offload would name it.
|
|
610
|
+
*/
|
|
611
|
+
export async function applyTwinWrites(service, writes, root) {
|
|
612
|
+
if (writes.length <= 1 || performsAtHeadNow(service, root) || writes.some((write) => write.idempotencyKey || write.actionId || write.preconditions?.length)) {
|
|
613
|
+
const out = [];
|
|
614
|
+
for (const write of writes)
|
|
615
|
+
out.push(await applyTwinWrite(service, write, root));
|
|
616
|
+
return out;
|
|
617
|
+
}
|
|
618
|
+
// Each distinct input offloaded once (S2 R17): `offloadInput` is content-addressed (a string's blob is named by its
|
|
619
|
+
// SHA-256), so the result is the one each write's own offload would give, and a command's writes, which share one
|
|
620
|
+
// recorded request, hash and look up its blob once instead of once per write.
|
|
621
|
+
const offloaded = new Map();
|
|
622
|
+
for (const write of writes)
|
|
623
|
+
if (write.input !== undefined && !offloaded.has(write.input))
|
|
624
|
+
offloaded.set(write.input, await offloadInput(service, write.input, root));
|
|
625
|
+
const committed = appendActionOccurrences(writes.map((write) => actionForTwinWrite(service, write.input === undefined ? write : { ...write, input: offloaded.get(write.input) })), root);
|
|
626
|
+
const out = [];
|
|
627
|
+
for (const [index, { appended, action: landed, placeholder }] of committed.entries()) {
|
|
628
|
+
const write = writes[index];
|
|
629
|
+
// the head does not perform here (`performsAtHeadNow` was false): this marks the request's deferred group as written
|
|
630
|
+
if (appended && !placeholder)
|
|
631
|
+
await performAtHead(service, landed, root);
|
|
632
|
+
const answerId = write.subjectId;
|
|
633
|
+
let resolved;
|
|
634
|
+
out.push({
|
|
635
|
+
result: { status: appended ? 'performed' : 'replayed', actionId: landed.id },
|
|
636
|
+
get resource() {
|
|
637
|
+
return resolved ??= readSubject(service, write.subjectType, answerId, root)
|
|
638
|
+
?? { id: answerId, type: write.subjectType, updatedAt: landed.occurredAt, ...write.fields };
|
|
639
|
+
},
|
|
640
|
+
});
|
|
641
|
+
}
|
|
642
|
+
return out;
|
|
643
|
+
}
|
|
604
644
|
export async function createTwinServer(options) {
|
|
605
645
|
const readOnly = options.readOnly ?? false;
|
|
606
646
|
const serveOptions = {
|
package/dist/src/storage.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { type WorldLockOptions } from './world-store.js';
|
|
1
2
|
import type { AppendEventResult, GenericWorldState, WorldPaths, WorldReducer, WorldServiceEvent } from './types.js';
|
|
2
3
|
/**
|
|
3
4
|
* Name of the per-project state directory holding world data
|
|
@@ -60,7 +61,7 @@ export declare function twinLog(kind: string, details: Record<string, unknown>):
|
|
|
60
61
|
* single-instance store it is a synchronous pass-through. Used to make read-then-append
|
|
61
62
|
* critical sections atomic. Public so world-runtime can guard concurrent `upWorld`
|
|
62
63
|
* claims of one instance dir with the SAME lock semantics the event log uses (TWIN-36). */
|
|
63
|
-
export declare function withFileLock<T>(lockPath: string, fn: () => T): T;
|
|
64
|
+
export declare function withFileLock<T>(lockPath: string, fn: () => T, options?: WorldLockOptions): T;
|
|
64
65
|
/** The parent log as observed rows: every entry the branch inherited or landed, as
|
|
65
66
|
* observed rows (log.ts `toEvent`). A branched world sees its base world's logs to the branch
|
|
66
67
|
* position first. */
|
package/dist/src/storage.js
CHANGED
|
@@ -3,7 +3,7 @@ import { dirname, isAbsolute, join, resolve } from 'node:path';
|
|
|
3
3
|
import { withStateLock, withStateRemoval } from "./ancestry.js";
|
|
4
4
|
import { appendParentEntry, toEntry } from "./log.js";
|
|
5
5
|
import { GenericWorldStateSchema, WorldServiceEventSchema, } from "./schemas.js";
|
|
6
|
-
import { getActiveWorldStore } from "./world-store.js";
|
|
6
|
+
import { getActiveWorldStore, storageLockScope, TREE_GENERATION } from "./world-store.js";
|
|
7
7
|
import { refuseReadOnlyWrite } from "./request-scope.js";
|
|
8
8
|
import { parentEntries, toEvent } from "./log.js";
|
|
9
9
|
function nowIso() {
|
|
@@ -181,8 +181,8 @@ function appendJsonl(path, value) {
|
|
|
181
181
|
* single-instance store it is a synchronous pass-through. Used to make read-then-append
|
|
182
182
|
* critical sections atomic. Public so world-runtime can guard concurrent `upWorld`
|
|
183
183
|
* claims of one instance dir with the SAME lock semantics the event log uses (TWIN-36). */
|
|
184
|
-
export function withFileLock(lockPath, fn) {
|
|
185
|
-
return withStateLock(lockPath, fn);
|
|
184
|
+
export function withFileLock(lockPath, fn, options = {}) {
|
|
185
|
+
return withStateLock(lockPath, fn, storageLockScope(options));
|
|
186
186
|
}
|
|
187
187
|
function writeJsonAtomic(path, value) {
|
|
188
188
|
getActiveWorldStore().writeAtomic(path, `${JSON.stringify(value, null, 2)}\n`);
|
|
@@ -327,6 +327,9 @@ function isKnownServiceEntry(name) {
|
|
|
327
327
|
// log.ts: the branch pointer and the checkpoint
|
|
328
328
|
if (name === 'branch.json' || name === 'checkpoints')
|
|
329
329
|
return true;
|
|
330
|
+
// world-store.ts: the generation a file store keeps beside a tree's files
|
|
331
|
+
if (name === TREE_GENERATION)
|
|
332
|
+
return true;
|
|
330
333
|
return KNOWN_SERVICE_ENTRIES.has(name) || name.endsWith('.tmp') || /\.stale\.\d+\.\d+$/.test(name);
|
|
331
334
|
}
|
|
332
335
|
/** A service dir "looks like" one of ours when every entry in it is something
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function worldBootAnswer(boot: string): Response;
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
// One private boot answer for the kernel's HTTP and gRPC serve seams.
|
|
2
|
+
// Reading process identity happens on request; importing this helper has no side effects.
|
|
3
|
+
export function worldBootAnswer(boot) {
|
|
4
|
+
const pid = globalThis.process?.pid;
|
|
5
|
+
return Response.json({ boot, pid });
|
|
6
|
+
}
|
package/dist/src/world-clock.js
CHANGED
|
@@ -23,10 +23,14 @@ const machineNow = () => {
|
|
|
23
23
|
* now); real wall-clock time otherwise. */
|
|
24
24
|
export function worldNow() {
|
|
25
25
|
const file = process.env[WORLD_CLOCK_ENV];
|
|
26
|
-
|
|
26
|
+
// One read answers both whether the clock file is there and what it holds.
|
|
27
|
+
// Kept while the clock file's own facts stand (world-store.ts readKept): every operation asks the time.
|
|
28
|
+
const store = getActiveWorldStore();
|
|
29
|
+
const held = file ? (store.readKept ? store.readKept(file) : store.read(file)) : null;
|
|
30
|
+
if (held !== null) {
|
|
27
31
|
// A configured-but-corrupt clock is a WORLD defect — parseClock fails loudly, never silently drifting back to
|
|
28
32
|
// wall-clock (two services disagreeing about now breaks the world).
|
|
29
|
-
const clock = clockForm.parseClock(
|
|
33
|
+
const clock = clockForm.parseClock(held);
|
|
30
34
|
return new Date(clockForm.clockNowMs(clock, machineNow())).toISOString();
|
|
31
35
|
}
|
|
32
36
|
return new Date(machineNow()).toISOString();
|