@volter/world-core 3.0.154 → 3.0.156
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/generated/pack-facts.json +140 -0
- package/dist/inject.cjs +150 -86
- package/dist/src/actions.d.ts +15 -0
- package/dist/src/actions.js +82 -21
- package/dist/src/ancestry.d.ts +5 -4
- package/dist/src/ancestry.js +53 -28
- package/dist/src/changeset.js +1 -1
- package/dist/src/clickhouse/compression.d.ts +28 -0
- package/dist/src/clickhouse/compression.js +281 -0
- package/dist/src/clickhouse/index.d.ts +1 -0
- package/dist/src/clickhouse/index.js +1 -0
- package/dist/src/client-bundle.js +15 -15
- package/dist/src/derived-core.d.ts +12 -1
- package/dist/src/derived-core.js +145 -11
- package/dist/src/derived.js +3 -0
- package/dist/src/fork.js +1 -1
- package/dist/src/graphql-wire.d.ts +5 -0
- package/dist/src/graphql-wire.js +6 -6
- package/dist/src/grpc-wire.js +2 -1
- package/dist/src/h2.js +63 -10
- package/dist/src/head.d.ts +3 -0
- package/dist/src/head.js +9 -0
- package/dist/src/history.d.ts +1 -1
- package/dist/src/history.js +7 -7
- package/dist/src/index.d.ts +4 -1
- package/dist/src/index.js +5 -1
- package/dist/src/log.d.ts +23 -3
- package/dist/src/log.js +160 -37
- package/dist/src/managed-original-inputs.d.ts +101 -0
- package/dist/src/managed-original-inputs.js +125 -0
- package/dist/src/mongodb/client.d.ts +8 -0
- package/dist/src/mongodb/client.js +27 -0
- package/dist/src/mongodb/index.d.ts +2 -0
- package/dist/src/mongodb/index.js +5 -0
- package/dist/src/mongodb/stream.d.ts +74 -0
- package/dist/src/mongodb/stream.js +206 -0
- package/dist/src/observe.js +1 -1
- package/dist/src/pack-fetch.d.ts +2 -1
- package/dist/src/pack-fetch.js +21 -9
- package/dist/src/packRegistry.d.ts +10 -14
- package/dist/src/packRegistry.js +2 -13
- package/dist/src/protocol.d.ts +13 -0
- package/dist/src/protocol.js +13 -0
- package/dist/src/redis/engine.d.ts +10 -4
- package/dist/src/redis/engine.js +103 -32
- package/dist/src/redis/member-storage.js +20 -8
- package/dist/src/redis/service-times.d.ts +25 -0
- package/dist/src/redis/service-times.js +87 -0
- package/dist/src/redis/stream.d.ts +2 -1
- package/dist/src/redis/stream.js +109 -20
- package/dist/src/runtime-state.d.ts +11 -0
- package/dist/src/runtime-state.js +13 -0
- package/dist/src/runtime.d.ts +4 -2
- package/dist/src/runtime.js +5 -2
- package/dist/src/serve-http.d.ts +2 -0
- package/dist/src/serve-http.js +21 -4
- package/dist/src/serve.d.ts +10 -0
- package/dist/src/serve.js +43 -3
- package/dist/src/storage.d.ts +2 -1
- package/dist/src/storage.js +6 -3
- package/dist/src/world-boot.d.ts +1 -0
- package/dist/src/world-boot.js +6 -0
- package/dist/src/world-clock.js +6 -2
- package/dist/src/world-store.d.ts +108 -7
- package/dist/src/world-store.js +883 -103
- package/dist/src/xml.d.ts +30 -0
- package/dist/src/xml.js +74 -0
- package/generated/pack-facts.json +140 -0
- package/inject.cjs +150 -86
- package/package.json +20 -1
- package/src/actions.ts +78 -21
- package/src/ancestry.ts +46 -24
- package/src/changeset.ts +1 -1
- package/src/clickhouse/compression.ts +172 -0
- package/src/clickhouse/index.ts +2 -0
- package/src/client-bundle.ts +13 -14
- package/src/derived-core.ts +142 -11
- package/src/derived.ts +3 -0
- package/src/fork.ts +1 -1
- package/src/graphql-wire.ts +9 -6
- package/src/grpc-wire.ts +2 -1
- package/src/h2.ts +46 -5
- package/src/head.ts +9 -0
- package/src/history.ts +7 -7
- package/src/index.ts +6 -1
- package/src/log.ts +147 -32
- package/src/managed-original-inputs.ts +163 -0
- package/src/mongodb/client.ts +27 -0
- package/src/mongodb/index.ts +5 -0
- package/src/mongodb/stream.ts +213 -0
- package/src/observe.ts +1 -1
- package/src/pack-fetch.ts +23 -11
- package/src/packRegistry.ts +3 -15
- package/src/protocol.ts +13 -0
- package/src/redis/engine.ts +100 -57
- package/src/redis/member-storage.ts +17 -9
- package/src/redis/service-times.ts +81 -0
- package/src/redis/stream.ts +66 -14
- package/src/runtime-state.ts +14 -0
- package/src/runtime.ts +6 -1
- package/src/serve-http.ts +23 -4
- package/src/serve.ts +45 -3
- package/src/storage.ts +5 -3
- package/src/world-boot.ts +6 -0
- package/src/world-clock.ts +6 -2
- package/src/world-store.ts +704 -91
- package/src/xml.ts +71 -0
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
export interface OriginalInputReference {
|
|
2
|
+
version: 1;
|
|
3
|
+
/** Relative to the needs file, or to the World configuration after init copies it. */
|
|
4
|
+
root: string;
|
|
5
|
+
manifest: "manifest.json";
|
|
6
|
+
sha256: string;
|
|
7
|
+
inventoryDigest: string;
|
|
8
|
+
}
|
|
9
|
+
export interface OriginalInputCapability {
|
|
10
|
+
version: 1;
|
|
11
|
+
manifestArg: "--original-inputs";
|
|
12
|
+
rootArg: "--original-root";
|
|
13
|
+
digestArg: "--original-inputs-sha256";
|
|
14
|
+
/** Image paths whose complete overlay contents the consumer requires. */
|
|
15
|
+
paths: string[];
|
|
16
|
+
}
|
|
17
|
+
export interface OriginalInputEntry {
|
|
18
|
+
target: string;
|
|
19
|
+
type: "file" | "directory" | "link";
|
|
20
|
+
mode: number;
|
|
21
|
+
bytes?: number;
|
|
22
|
+
sha256?: string;
|
|
23
|
+
object?: string;
|
|
24
|
+
link?: string;
|
|
25
|
+
source: {
|
|
26
|
+
kind: "image" | "project";
|
|
27
|
+
path: string;
|
|
28
|
+
layer?: string;
|
|
29
|
+
blob?: string;
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
export interface ManagedOriginalInputs {
|
|
33
|
+
kind: "volter.managed-original-inputs";
|
|
34
|
+
version: 1;
|
|
35
|
+
service: string;
|
|
36
|
+
family: string;
|
|
37
|
+
/** Canonical selected service declaration; used to refuse a different stack on reuse. */
|
|
38
|
+
declaration: string;
|
|
39
|
+
source: {
|
|
40
|
+
image: string;
|
|
41
|
+
manifest: string;
|
|
42
|
+
config: string;
|
|
43
|
+
layers: string[];
|
|
44
|
+
commit: string;
|
|
45
|
+
compose: string;
|
|
46
|
+
composeSha256: string;
|
|
47
|
+
composeBytes: number;
|
|
48
|
+
composeObject: string;
|
|
49
|
+
};
|
|
50
|
+
/** Authenticated complete image metadata, before the declared mounts overlay it. */
|
|
51
|
+
imageInventory: Array<{
|
|
52
|
+
target: string;
|
|
53
|
+
type: string;
|
|
54
|
+
mode: number;
|
|
55
|
+
bytes: number;
|
|
56
|
+
layer: string;
|
|
57
|
+
link?: string;
|
|
58
|
+
}>;
|
|
59
|
+
imageInventoryDigest: string;
|
|
60
|
+
projectInventory: Array<{
|
|
61
|
+
path: string;
|
|
62
|
+
mode: number;
|
|
63
|
+
blob: string;
|
|
64
|
+
}>;
|
|
65
|
+
projectInventoryDigest: string;
|
|
66
|
+
roots: string[];
|
|
67
|
+
/** Complete contents beneath roots, plus every original project bind mount. */
|
|
68
|
+
inventory: OriginalInputEntry[];
|
|
69
|
+
inventoryDigest: string;
|
|
70
|
+
start: {
|
|
71
|
+
imageConfig: Record<string, unknown>;
|
|
72
|
+
service: Record<string, unknown>;
|
|
73
|
+
imageEntrypoint: string[] | null;
|
|
74
|
+
imageCommand: string[] | null;
|
|
75
|
+
command?: string[];
|
|
76
|
+
entrypoint?: string[];
|
|
77
|
+
environment: Record<string, string>;
|
|
78
|
+
envFiles: Array<{
|
|
79
|
+
path: string;
|
|
80
|
+
bytes: number;
|
|
81
|
+
sha256: string;
|
|
82
|
+
object: string;
|
|
83
|
+
}>;
|
|
84
|
+
unset: string[];
|
|
85
|
+
selfHosts: string[];
|
|
86
|
+
mounts: Array<{
|
|
87
|
+
kind: string;
|
|
88
|
+
source?: string;
|
|
89
|
+
target: string;
|
|
90
|
+
readOnly: boolean;
|
|
91
|
+
}>;
|
|
92
|
+
};
|
|
93
|
+
}
|
|
94
|
+
export declare const originalTarget: (path: string) => boolean;
|
|
95
|
+
export declare const originalRelative: (path: string) => boolean;
|
|
96
|
+
export declare function referenceOf(value: unknown): OriginalInputReference;
|
|
97
|
+
/** Hash and validate neutral carrier data. The caller supplies I/O; no pack/vendor behavior or import effects. */
|
|
98
|
+
export declare function readSnapshot(bytes: Uint8Array, expected: string, readObject: (path: string, bytes: number) => Uint8Array): {
|
|
99
|
+
manifest: ManagedOriginalInputs;
|
|
100
|
+
objects: ReadonlyMap<string, Uint8Array>;
|
|
101
|
+
};
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
/** Source-bound files at a managed replacement. No vendor interpretation or host paths. */
|
|
2
|
+
import * as nodeCrypto from 'node:crypto';
|
|
3
|
+
const digest = (algorithm, value) => nodeCrypto.createHash(algorithm).update(value).digest('hex');
|
|
4
|
+
export const originalTarget = (path) => path === "/" || (path.startsWith("/") && !path.slice(1).split("/").some(part => !part || part === "." || part === ".." || /[\0\r\n\\]/u.test(part)));
|
|
5
|
+
export const originalRelative = (path) => !!path && !path.split("/").some(part => !part || part === "." || part === ".." || /[\0\r\n\\:]/u.test(part));
|
|
6
|
+
const hex = (value) => typeof value === 'string' && /^[a-f0-9]{64}$/.test(value);
|
|
7
|
+
const oci = (value) => typeof value === 'string' && /^sha256:[a-f0-9]{64}$/.test(value);
|
|
8
|
+
const bytesOf = (value) => Number.isSafeInteger(value) && Number(value) >= 0;
|
|
9
|
+
const strings = (value) => Array.isArray(value) && value.every(entry => typeof entry === 'string' && !/[\0\r\n]/.test(entry));
|
|
10
|
+
const bound = 128 * 1024 * 1024;
|
|
11
|
+
export function referenceOf(value) {
|
|
12
|
+
if (!value || typeof value !== 'object' || Array.isArray(value))
|
|
13
|
+
throw new Error('Original inputs need a versioned reference');
|
|
14
|
+
const ref = value;
|
|
15
|
+
if (ref.version !== 1 || ref.manifest !== 'manifest.json' || typeof ref.root !== 'string' || !originalRelative(ref.root) || !hex(ref.sha256) || !hex(ref.inventoryDigest)
|
|
16
|
+
|| Object.keys(ref).some(key => !['version', 'root', 'manifest', 'sha256', 'inventoryDigest'].includes(key)))
|
|
17
|
+
throw new Error('Invalid original-input reference');
|
|
18
|
+
return { ...ref };
|
|
19
|
+
}
|
|
20
|
+
/** Hash and validate neutral carrier data. The caller supplies I/O; no pack/vendor behavior or import effects. */
|
|
21
|
+
export function readSnapshot(bytes, expected, readObject) {
|
|
22
|
+
if (!(bytes instanceof Uint8Array) || bytes.length > 64 * 1024 * 1024 || !hex(expected) || digest('sha256', bytes) !== expected)
|
|
23
|
+
throw new Error('Original-input manifest identity differs');
|
|
24
|
+
const manifest = JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(bytes));
|
|
25
|
+
const source = manifest.source, start = manifest.start;
|
|
26
|
+
if (manifest.kind !== 'volter.managed-original-inputs' || manifest.version !== 1 || typeof manifest.service !== 'string' || !manifest.service || typeof manifest.family !== 'string' || !/^[a-z][a-z0-9-]*$/.test(manifest.family)
|
|
27
|
+
|| typeof manifest.declaration !== 'string' || !source || !/^[a-f0-9]{40}$/.test(source.commit) || !/^[A-Za-z0-9./:_-]+@sha256:[a-f0-9]{64}$/.test(source.image)
|
|
28
|
+
|| !oci(source.manifest) || !oci(source.config) || !Array.isArray(source.layers) || !source.layers.every(oci) || !originalRelative(source.compose)
|
|
29
|
+
|| !Array.isArray(manifest.imageInventory) || !Array.isArray(manifest.projectInventory) || !Array.isArray(manifest.inventory) || !strings(manifest.roots) || !manifest.roots.length || !manifest.roots.every(originalTarget) || new Set(manifest.roots).size !== manifest.roots.length
|
|
30
|
+
|| !hex(manifest.imageInventoryDigest) || digest('sha256', JSON.stringify(manifest.imageInventory)) !== manifest.imageInventoryDigest
|
|
31
|
+
|| !hex(manifest.inventoryDigest) || digest('sha256', JSON.stringify(manifest.inventory)) !== manifest.inventoryDigest
|
|
32
|
+
|| !hex(manifest.projectInventoryDigest) || digest('sha256', JSON.stringify(manifest.projectInventory)) !== manifest.projectInventoryDigest
|
|
33
|
+
|| !start || !start.imageConfig || typeof start.imageConfig !== 'object' || Array.isArray(start.imageConfig) || !start.service || typeof start.service !== 'object' || Array.isArray(start.service)
|
|
34
|
+
|| !strings(start.selfHosts) || !start.selfHosts.length || !strings(start.unset) || !Array.isArray(start.envFiles) || !Array.isArray(start.mounts)
|
|
35
|
+
|| !start.environment || typeof start.environment !== 'object' || Array.isArray(start.environment) || Object.entries(start.environment).some(([key, value]) => !/^[A-Za-z_][A-Za-z0-9_]*$/.test(key) || typeof value !== 'string')
|
|
36
|
+
|| [start.imageEntrypoint, start.imageCommand].some(value => value !== null && !strings(value)) || [start.command, start.entrypoint].some(value => value !== undefined && !strings(value)))
|
|
37
|
+
throw new Error('Invalid original-input manifest');
|
|
38
|
+
if (JSON.stringify(JSON.parse(manifest.declaration)) !== JSON.stringify(start.service) || start.service.name !== manifest.service
|
|
39
|
+
|| JSON.stringify(start.command) !== JSON.stringify(start.service.command) || JSON.stringify(start.entrypoint) !== JSON.stringify(start.service.entrypoint)
|
|
40
|
+
|| JSON.stringify(start.imageEntrypoint) !== JSON.stringify(start.imageConfig.Entrypoint ?? null) || JSON.stringify(start.imageCommand) !== JSON.stringify(start.imageConfig.Cmd ?? null)
|
|
41
|
+
|| !Array.isArray(start.service.volumes)
|
|
42
|
+
|| JSON.stringify(start.mounts.map(mount => [mount.kind, mount.source ?? null, mount.target, mount.readOnly])) !== JSON.stringify(start.service.volumes.map(mount => [mount.kind, mount.source ?? null, mount.target, mount.readOnly])))
|
|
43
|
+
throw new Error('Original-input start declaration differs');
|
|
44
|
+
const project = new Map();
|
|
45
|
+
for (const entry of manifest.projectInventory) {
|
|
46
|
+
if (!entry || !originalRelative(entry.path) || project.has(entry.path) || ![0o100644, 0o100755, 0o120000, 0o160000].includes(entry.mode) || !/^[a-f0-9]{40}$/.test(entry.blob))
|
|
47
|
+
throw new Error('Invalid original project inventory');
|
|
48
|
+
project.set(entry.path, entry);
|
|
49
|
+
}
|
|
50
|
+
const image = new Map();
|
|
51
|
+
for (const entry of manifest.imageInventory) {
|
|
52
|
+
if (!entry || !originalTarget(entry.target) || image.has(entry.target) || !['file', 'directory', 'link', 'other'].includes(entry.type) || !Number.isInteger(entry.mode) || entry.mode < 0 || entry.mode > 0o7777 || !bytesOf(entry.bytes) || !source.layers.includes(entry.layer)
|
|
53
|
+
|| (entry.type === 'link' && (typeof entry.link !== 'string' || /[\0\r\n\\]/.test(entry.link))))
|
|
54
|
+
throw new Error('Invalid original image inventory');
|
|
55
|
+
image.set(entry.target, entry);
|
|
56
|
+
}
|
|
57
|
+
const targets = new Set(), selected = new Map(), expectedObjects = new Map();
|
|
58
|
+
const hold = (entry) => {
|
|
59
|
+
if (!hex(entry.sha256) || !bytesOf(entry.bytes) || entry.object !== `objects/${entry.sha256}` || (expectedObjects.has(entry.sha256) && expectedObjects.get(entry.sha256) !== entry.bytes))
|
|
60
|
+
throw new Error('Invalid original-input payload identity');
|
|
61
|
+
expectedObjects.set(entry.sha256, entry.bytes);
|
|
62
|
+
};
|
|
63
|
+
for (const entry of manifest.inventory) {
|
|
64
|
+
if (!entry || !originalTarget(entry.target) || targets.has(entry.target) || !['file', 'directory', 'link'].includes(entry.type) || !Number.isInteger(entry.mode) || entry.mode < 0 || entry.mode > 0o7777 || !entry.source || !['image', 'project'].includes(entry.source.kind))
|
|
65
|
+
throw new Error('Invalid original-input inventory');
|
|
66
|
+
targets.add(entry.target);
|
|
67
|
+
selected.set(entry.target, entry);
|
|
68
|
+
if (entry.source.kind === 'image') {
|
|
69
|
+
const original = image.get(entry.source.path);
|
|
70
|
+
if (!original || original.layer !== entry.source.layer || original.type !== entry.type || original.mode !== entry.mode || (entry.type === 'file' && original.bytes !== entry.bytes) || (entry.type === 'link' && original.link !== entry.link))
|
|
71
|
+
throw new Error('Original-input image provenance differs');
|
|
72
|
+
}
|
|
73
|
+
else {
|
|
74
|
+
const original = project.get(entry.source.path);
|
|
75
|
+
if (!originalRelative(entry.source.path) || (entry.type === 'file' && (!original || original.blob !== entry.source.blob || (original.mode & 0o777) !== entry.mode)))
|
|
76
|
+
throw new Error('Original-input project provenance differs');
|
|
77
|
+
}
|
|
78
|
+
if (entry.type === 'file')
|
|
79
|
+
hold(entry);
|
|
80
|
+
else if (entry.object !== undefined || entry.bytes !== undefined || entry.sha256 !== undefined || (entry.type === 'link' && (typeof entry.link !== 'string' || /[\0\r\n\\]/.test(entry.link))))
|
|
81
|
+
throw new Error('Invalid original-input non-file');
|
|
82
|
+
}
|
|
83
|
+
for (const mount of start.mounts)
|
|
84
|
+
if (!mount || !['bind', 'named', 'anonymous'].includes(mount.kind) || !originalTarget(mount.target) || typeof mount.readOnly !== 'boolean' || (mount.source !== undefined && typeof mount.source !== 'string'))
|
|
85
|
+
throw new Error('Invalid original-input mount declaration');
|
|
86
|
+
const roots = new Set(manifest.roots), mounted = new Set(start.mounts.filter(mount => mount.kind === 'bind').map(mount => mount.target));
|
|
87
|
+
const under = (path, roots) => {
|
|
88
|
+
for (let at = path;; at = at.slice(0, at.lastIndexOf('/')) || '/') {
|
|
89
|
+
if (roots.has(at))
|
|
90
|
+
return true;
|
|
91
|
+
if (at === '/')
|
|
92
|
+
return false;
|
|
93
|
+
}
|
|
94
|
+
};
|
|
95
|
+
for (const entry of image.values())
|
|
96
|
+
if (under(entry.target, roots) && !under(entry.target, mounted)) {
|
|
97
|
+
const captured = selected.get(entry.target);
|
|
98
|
+
if (!captured || captured.source.kind !== 'image' || captured.source.path !== entry.target)
|
|
99
|
+
throw new Error('Original image input was omitted from its declared root');
|
|
100
|
+
}
|
|
101
|
+
hold({ sha256: source.composeSha256, bytes: source.composeBytes, object: source.composeObject });
|
|
102
|
+
for (const file of start.envFiles) {
|
|
103
|
+
if (!file || !originalRelative(file.path))
|
|
104
|
+
throw new Error('Invalid original-input env_file');
|
|
105
|
+
hold(file);
|
|
106
|
+
}
|
|
107
|
+
if ([...expectedObjects.values()].reduce((sum, bytes) => sum + bytes, 0) > bound)
|
|
108
|
+
throw new Error('Original-input snapshot exceeds its byte bound');
|
|
109
|
+
const objects = new Map();
|
|
110
|
+
for (const [sha256, size] of expectedObjects) {
|
|
111
|
+
const data = readObject(`objects/${sha256}`, size);
|
|
112
|
+
if (!(data instanceof Uint8Array) || data.length !== size || digest('sha256', data) !== sha256)
|
|
113
|
+
throw new Error('Original-input payload bytes differ');
|
|
114
|
+
objects.set(sha256, data.slice());
|
|
115
|
+
}
|
|
116
|
+
for (const entry of manifest.inventory)
|
|
117
|
+
if (entry.type === 'file' && entry.source.kind === 'project') {
|
|
118
|
+
const data = objects.get(entry.sha256), header = new TextEncoder().encode(`blob ${data.length}\0`), blob = new Uint8Array(header.length + data.length);
|
|
119
|
+
blob.set(header);
|
|
120
|
+
blob.set(data, header.length);
|
|
121
|
+
if (digest('sha1', blob) !== entry.source.blob)
|
|
122
|
+
throw new Error('Original-input project blob identity differs');
|
|
123
|
+
}
|
|
124
|
+
return { manifest, objects };
|
|
125
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { type MongoCodec, type MongoDocument } from './stream.js';
|
|
2
|
+
/** Sends bytes and returns every byte the server answered to them. */
|
|
3
|
+
export type MongoExchange = (bytes: Uint8Array) => Promise<Uint8Array>;
|
|
4
|
+
/** A client on one connection: `command` sends a document to `database` and resolves with the reply document,
|
|
5
|
+
* whatever its `ok`. A reply that is not one whole OP_MSG answering this request is an error. */
|
|
6
|
+
export declare function mongoClient(exchange: MongoExchange, codec: MongoCodec): {
|
|
7
|
+
command(database: string, command: MongoDocument): Promise<MongoDocument>;
|
|
8
|
+
};
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
// A MongoDB client over any byte exchange: the other side of ./stream.ts, so a journey speaks the wire as an
|
|
2
|
+
// application's driver does. It sends one command document as an OP_MSG and returns the document the server
|
|
3
|
+
// answered; it reads what the protocol answers and decides nothing.
|
|
4
|
+
// source: archive:https://registry.npmjs.org/mongodb/-/mongodb-6.20.0.tgz#sha256=a12bd599cb4f39cdf222b32380ea3efd1f60c6d88615e0700632a303a9006b02!/package/src/cmap/commands.ts "moreToCome is an OP_MSG only concept"
|
|
5
|
+
import { MongoOp, decodeMsg, encodeMsg } from "./stream.js";
|
|
6
|
+
/** A client on one connection: `command` sends a document to `database` and resolves with the reply document,
|
|
7
|
+
* whatever its `ok`. A reply that is not one whole OP_MSG answering this request is an error. */
|
|
8
|
+
export function mongoClient(exchange, codec) {
|
|
9
|
+
let nextRequest = 0;
|
|
10
|
+
return {
|
|
11
|
+
async command(database, command) {
|
|
12
|
+
const requestId = ++nextRequest;
|
|
13
|
+
const answered = await exchange(encodeMsg(codec, { ...command, $db: database }, requestId, 0));
|
|
14
|
+
if (answered.length < 16)
|
|
15
|
+
throw new Error('the server answered no message');
|
|
16
|
+
const view = new DataView(answered.buffer, answered.byteOffset, answered.byteLength);
|
|
17
|
+
const length = view.getInt32(0, true), responseTo = view.getInt32(8, true), opcode = view.getInt32(12, true);
|
|
18
|
+
if (length !== answered.length)
|
|
19
|
+
throw new Error(`the server answered ${answered.length} bytes, and its message says ${length}`);
|
|
20
|
+
if (opcode !== MongoOp.MSG)
|
|
21
|
+
throw new Error(`the server answered opcode ${opcode}, not OP_MSG`);
|
|
22
|
+
if (responseTo !== requestId)
|
|
23
|
+
throw new Error(`the server answered request ${responseTo}, not ${requestId}`);
|
|
24
|
+
return decodeMsg(codec, answered).document;
|
|
25
|
+
},
|
|
26
|
+
};
|
|
27
|
+
}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
// THE KERNEL'S MONGODB WIRE (`@volter/world-core/mongodb`): MongoDB's message framing as a TwinStream, and its client.
|
|
2
|
+
// A pack that serves MongoDB passes what a command means (its engine) and the document codec; the kernel names no vendor.
|
|
3
|
+
// docs/contributing/architecture.md, "Other wires", the MongoDB wire.
|
|
4
|
+
export * from "./stream.js";
|
|
5
|
+
export * from "./client.js";
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import type { TwinStream } from '../twin-fetch.js';
|
|
2
|
+
/** A command or reply document as the pack's codec gives it: the pack decides how BSON's types appear in it. */
|
|
3
|
+
export type MongoDocument = Record<string, unknown>;
|
|
4
|
+
/** BSON, as the pack supplies it (the `bson` library, installed by the pack's package.json). The kernel frames
|
|
5
|
+
* documents and never reads inside one but for a command's `$db`. */
|
|
6
|
+
export type MongoCodec = {
|
|
7
|
+
serialize(document: MongoDocument): Uint8Array;
|
|
8
|
+
deserialize(bytes: Uint8Array): MongoDocument;
|
|
9
|
+
};
|
|
10
|
+
/** What the sender's message said beside its command: whether it will take more than one reply to it (OP_MSG's exhaustAllowed). */
|
|
11
|
+
export type MongoAsked = {
|
|
12
|
+
exhaustAllowed: boolean;
|
|
13
|
+
};
|
|
14
|
+
declare const TIMED: unique symbol;
|
|
15
|
+
/**
|
|
16
|
+
* An answer the wire sends LATER, or goes on sending: the vendor's protocol does this in real time (an awaitable
|
|
17
|
+
* `hello` is held for the sender's maxAwaitTimeMS, and one sent with exhaustAllowed is answered again every such
|
|
18
|
+
* interval, each reply marked moreToCome). The pack says WHAT and WHEN as data and holds no clock: the same request
|
|
19
|
+
* gets the same answer every time. The wire does the holding and the repeating on the wall clock, and the delay is
|
|
20
|
+
* never read back into state (architecture, "The MongoDB wire").
|
|
21
|
+
*/
|
|
22
|
+
export type MongoTimedReply = {
|
|
23
|
+
readonly [TIMED]: true;
|
|
24
|
+
reply: MongoDocument;
|
|
25
|
+
afterMs: number;
|
|
26
|
+
everyMs?: number;
|
|
27
|
+
};
|
|
28
|
+
/**
|
|
29
|
+
* `reply` after `afterMs` (0: at once); with `everyMs`, again each `everyMs` until the peer closes, where the sender
|
|
30
|
+
* allowed more than one reply. The two differ in the vendor's own answers: a hello naming the server's current
|
|
31
|
+
* topology is first answered after the wait, one naming another process at once, and both then repeat at the wait.
|
|
32
|
+
*/
|
|
33
|
+
export declare const timedReply: (reply: MongoDocument, afterMs: number, everyMs?: number) => MongoTimedReply;
|
|
34
|
+
export type MongoStreamOptions<Session> = {
|
|
35
|
+
codec: MongoCodec;
|
|
36
|
+
/** A connection opened: what the pack keeps for it (its sessions, an open transaction); never the tree. */
|
|
37
|
+
open(peer: string): Session;
|
|
38
|
+
/** Answer ONE command document sent to `database`. A document sequence (an insert's `documents`, an update's
|
|
39
|
+
* `updates`) arrives as that field holding the array. The answer is the reply document, `ok: 1` or the vendor's
|
|
40
|
+
* error document; a command the pack does not serve is the pack's to answer as the vendor answers an unknown one. */
|
|
41
|
+
command(session: Session, database: string, command: MongoDocument, asked: MongoAsked): Promise<MongoDocument | MongoTimedReply>;
|
|
42
|
+
/** The connection went away: abort what it left open. */
|
|
43
|
+
close?(session: Session): void;
|
|
44
|
+
/** Told why a connection is ended by this stream and not by its peer: the pack threw answering a command, or a
|
|
45
|
+
* message was one this wire does not serve. Ending the connection is all the wire can say to the client, so the
|
|
46
|
+
* reason is said here; unset, it is written to the process's error output. A stream that ends connections in
|
|
47
|
+
* silence leaves its operator with a client that hangs and nothing to read. */
|
|
48
|
+
ended?(reason: string, peer: string): void;
|
|
49
|
+
/** The largest message accepted; the vendor's server names 48,000,000 in its hello (`maxMessageSizeBytes`). */
|
|
50
|
+
maxMessageSizeBytes?: number;
|
|
51
|
+
};
|
|
52
|
+
export declare const MongoOp: {
|
|
53
|
+
readonly REPLY: 1;
|
|
54
|
+
readonly QUERY: 2004;
|
|
55
|
+
readonly MSG: 2013;
|
|
56
|
+
};
|
|
57
|
+
/** OP_MSG's flag bits. */
|
|
58
|
+
export declare const MongoMsgFlag: {
|
|
59
|
+
readonly CHECKSUM_PRESENT: number;
|
|
60
|
+
readonly MORE_TO_COME: number;
|
|
61
|
+
readonly EXHAUST_ALLOWED: number;
|
|
62
|
+
};
|
|
63
|
+
/** An OP_MSG message carrying one document as its body section. */
|
|
64
|
+
export declare function encodeMsg(codec: MongoCodec, document: MongoDocument, requestId: number, responseTo: number, flags?: number): Uint8Array;
|
|
65
|
+
/** The command of an OP_MSG message: its body section, with each document sequence as the field it names. */
|
|
66
|
+
export declare function decodeMsg(codec: MongoCodec, message: Uint8Array): {
|
|
67
|
+
flags: number;
|
|
68
|
+
document: MongoDocument;
|
|
69
|
+
};
|
|
70
|
+
/** MongoDB's wire, served: each command document answered by the pack, in arrival order, on one connection. */
|
|
71
|
+
export declare function createMongoStream<Session>(options: MongoStreamOptions<Session>): TwinStream;
|
|
72
|
+
/** The bytes of a C string, for a client's OP_QUERY. */
|
|
73
|
+
export declare const cstring: (text: string) => Uint8Array;
|
|
74
|
+
export {};
|
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
const TIMED = Symbol('mongodb timed reply');
|
|
2
|
+
/**
|
|
3
|
+
* `reply` after `afterMs` (0: at once); with `everyMs`, again each `everyMs` until the peer closes, where the sender
|
|
4
|
+
* allowed more than one reply. The two differ in the vendor's own answers: a hello naming the server's current
|
|
5
|
+
* topology is first answered after the wait, one naming another process at once, and both then repeat at the wait.
|
|
6
|
+
*/
|
|
7
|
+
export const timedReply = (reply, afterMs, everyMs) => ({ [TIMED]: true, reply, afterMs, ...(everyMs !== undefined ? { everyMs } : {}) });
|
|
8
|
+
const isTimed = (answer) => answer[TIMED] === true;
|
|
9
|
+
export const MongoOp = { REPLY: 1, QUERY: 2004, MSG: 2013 };
|
|
10
|
+
/** OP_MSG's flag bits. */
|
|
11
|
+
export const MongoMsgFlag = { CHECKSUM_PRESENT: 1 << 0, MORE_TO_COME: 1 << 1, EXHAUST_ALLOWED: 1 << 16 };
|
|
12
|
+
const HEADER = 16;
|
|
13
|
+
const dec = new TextDecoder();
|
|
14
|
+
const enc = new TextEncoder();
|
|
15
|
+
const view = (bytes) => new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
16
|
+
function concat(parts) {
|
|
17
|
+
const out = new Uint8Array(parts.reduce((n, p) => n + p.length, 0));
|
|
18
|
+
let at = 0;
|
|
19
|
+
for (const p of parts) {
|
|
20
|
+
out.set(p, at);
|
|
21
|
+
at += p.length;
|
|
22
|
+
}
|
|
23
|
+
return out;
|
|
24
|
+
}
|
|
25
|
+
function header(length, requestId, responseTo, opcode) {
|
|
26
|
+
const out = new Uint8Array(HEADER), v = view(out);
|
|
27
|
+
v.setInt32(0, length, true);
|
|
28
|
+
v.setInt32(4, requestId, true);
|
|
29
|
+
v.setInt32(8, responseTo, true);
|
|
30
|
+
v.setInt32(12, opcode, true);
|
|
31
|
+
return out;
|
|
32
|
+
}
|
|
33
|
+
/** An OP_MSG message carrying one document as its body section. */
|
|
34
|
+
export function encodeMsg(codec, document, requestId, responseTo, flags = 0) {
|
|
35
|
+
const body = codec.serialize(document), bits = new Uint8Array(5);
|
|
36
|
+
view(bits).setUint32(0, flags, true);
|
|
37
|
+
return concat([header(HEADER + 4 + 1 + body.length, requestId, responseTo, MongoOp.MSG), bits, body]);
|
|
38
|
+
}
|
|
39
|
+
/** The command of an OP_MSG message: its body section, with each document sequence as the field it names. */
|
|
40
|
+
export function decodeMsg(codec, message) {
|
|
41
|
+
const v = view(message), flags = v.getUint32(HEADER, true), end = message.length - (flags & MongoMsgFlag.CHECKSUM_PRESENT ? 4 : 0);
|
|
42
|
+
let document;
|
|
43
|
+
const sequences = {};
|
|
44
|
+
for (let at = HEADER + 4; at < end;) {
|
|
45
|
+
const kind = message[at];
|
|
46
|
+
at += 1;
|
|
47
|
+
if (kind === 0) {
|
|
48
|
+
const size = v.getInt32(at, true);
|
|
49
|
+
if (size < 5 || at + size > end)
|
|
50
|
+
throw new Error('OP_MSG body section runs past the message');
|
|
51
|
+
document = codec.deserialize(message.subarray(at, at + size));
|
|
52
|
+
at += size;
|
|
53
|
+
}
|
|
54
|
+
else if (kind === 1) {
|
|
55
|
+
const size = v.getInt32(at, true), stop = at + size;
|
|
56
|
+
if (size < 4 || stop > end)
|
|
57
|
+
throw new Error('OP_MSG document sequence runs past the message');
|
|
58
|
+
const nameEnd = message.indexOf(0, at + 4);
|
|
59
|
+
if (nameEnd < 0 || nameEnd >= stop)
|
|
60
|
+
throw new Error('OP_MSG document sequence names no field');
|
|
61
|
+
const name = dec.decode(message.subarray(at + 4, nameEnd)), documents = [];
|
|
62
|
+
for (let cursor = nameEnd + 1; cursor < stop;) {
|
|
63
|
+
const length = v.getInt32(cursor, true);
|
|
64
|
+
if (length < 5 || cursor + length > stop)
|
|
65
|
+
throw new Error('OP_MSG document sequence holds a document that runs past it');
|
|
66
|
+
documents.push(codec.deserialize(message.subarray(cursor, cursor + length)));
|
|
67
|
+
cursor += length;
|
|
68
|
+
}
|
|
69
|
+
sequences[name] = documents;
|
|
70
|
+
at = stop;
|
|
71
|
+
}
|
|
72
|
+
else
|
|
73
|
+
throw new Error(`OP_MSG section kind ${kind} is unknown`);
|
|
74
|
+
}
|
|
75
|
+
if (!document)
|
|
76
|
+
throw new Error('OP_MSG carries no body section');
|
|
77
|
+
return { flags, document: { ...document, ...sequences } };
|
|
78
|
+
}
|
|
79
|
+
/** MongoDB's wire, served: each command document answered by the pack, in arrival order, on one connection. */
|
|
80
|
+
export function createMongoStream(options) {
|
|
81
|
+
const limit = options.maxMessageSizeBytes ?? 48_000_000;
|
|
82
|
+
let nextResponse = 0;
|
|
83
|
+
return (sink, peer) => {
|
|
84
|
+
const session = options.open(peer);
|
|
85
|
+
let buffer = new Uint8Array(0);
|
|
86
|
+
let closed = false;
|
|
87
|
+
let chain = Promise.resolve();
|
|
88
|
+
// A reply being held or repeated (a timed reply): its timer, and the messages that arrived meanwhile. A
|
|
89
|
+
// connection serves one command at a time, so nothing else is answered until a held reply is sent, and nothing
|
|
90
|
+
// at all while a repeating one runs (recording:hello-awaitable-exhaust-then-command.json: a ping sent into the
|
|
91
|
+
// stream has no reply). The timer is not work: `settled` does not wait for it, it does not keep the process
|
|
92
|
+
// alive (unref), and a close from either side clears it at once.
|
|
93
|
+
let held;
|
|
94
|
+
const parked = [];
|
|
95
|
+
const release = () => { if (held !== undefined) {
|
|
96
|
+
clearTimeout(held);
|
|
97
|
+
held = undefined;
|
|
98
|
+
} parked.length = 0; };
|
|
99
|
+
const hold = (ms, then) => {
|
|
100
|
+
held = setTimeout(() => { held = undefined; if (!closed)
|
|
101
|
+
then(); }, Math.max(0, ms));
|
|
102
|
+
held.unref?.();
|
|
103
|
+
};
|
|
104
|
+
const end = (reason) => {
|
|
105
|
+
if (closed)
|
|
106
|
+
return;
|
|
107
|
+
closed = true;
|
|
108
|
+
release();
|
|
109
|
+
(options.ended ?? ((why, from) => console.error(`mongodb wire: ended the connection from ${from}: ${why}`)))(reason, peer);
|
|
110
|
+
options.close?.(session);
|
|
111
|
+
sink.end();
|
|
112
|
+
};
|
|
113
|
+
const answer = async (message) => {
|
|
114
|
+
const v = view(message), requestId = v.getInt32(4, true), opcode = v.getInt32(12, true);
|
|
115
|
+
if (opcode === MongoOp.MSG) {
|
|
116
|
+
const { flags, document } = decodeMsg(options.codec, message);
|
|
117
|
+
const database = typeof document.$db === 'string' ? document.$db : 'admin';
|
|
118
|
+
const exhaustAllowed = (flags & MongoMsgFlag.EXHAUST_ALLOWED) !== 0;
|
|
119
|
+
const answered = await options.command(session, database, document, { exhaustAllowed });
|
|
120
|
+
// the sender waits for no reply to a message it marked moreToCome
|
|
121
|
+
if (flags & MongoMsgFlag.MORE_TO_COME)
|
|
122
|
+
return;
|
|
123
|
+
if (!isTimed(answered)) {
|
|
124
|
+
sink.write(encodeMsg(options.codec, answered, ++nextResponse, requestId));
|
|
125
|
+
return;
|
|
126
|
+
}
|
|
127
|
+
if (answered.everyMs === undefined || !exhaustAllowed) {
|
|
128
|
+
// held once: one reply after the wait, with no flag; then what arrived meanwhile is answered in order
|
|
129
|
+
// source: recording:hello-awaitable-no-exhaust.json "received"
|
|
130
|
+
hold(answered.afterMs, () => {
|
|
131
|
+
sink.write(encodeMsg(options.codec, answered.reply, ++nextResponse, requestId));
|
|
132
|
+
for (const message of parked.splice(0))
|
|
133
|
+
take(message);
|
|
134
|
+
});
|
|
135
|
+
return;
|
|
136
|
+
}
|
|
137
|
+
// repeated: each reply marked moreToCome; the first answers the request, each later one answers this
|
|
138
|
+
// wire's own previous reply, until the peer closes
|
|
139
|
+
// source: recording:hello-awaitable-exhaust.json "received"
|
|
140
|
+
let answers = requestId;
|
|
141
|
+
const again = () => {
|
|
142
|
+
const id = ++nextResponse;
|
|
143
|
+
sink.write(encodeMsg(options.codec, answered.reply, id, answers, MongoMsgFlag.MORE_TO_COME));
|
|
144
|
+
answers = id;
|
|
145
|
+
hold(answered.everyMs, again);
|
|
146
|
+
};
|
|
147
|
+
// source: recording:hello-stale-process-exhaust.json "received" (the first at once, the rest at the wait)
|
|
148
|
+
hold(answered.afterMs, again);
|
|
149
|
+
return;
|
|
150
|
+
}
|
|
151
|
+
if (opcode === MongoOp.QUERY) {
|
|
152
|
+
// the legacy form a driver's first hello may take: a query on `<database>.$cmd` whose document is the command
|
|
153
|
+
const nameEnd = message.indexOf(0, HEADER + 4);
|
|
154
|
+
if (nameEnd < 0)
|
|
155
|
+
throw new Error('OP_QUERY names no collection');
|
|
156
|
+
const namespace = dec.decode(message.subarray(HEADER + 4, nameEnd));
|
|
157
|
+
if (!namespace.endsWith('.$cmd'))
|
|
158
|
+
throw new Error(`OP_QUERY on ${namespace}: only a command on <database>.$cmd is served`);
|
|
159
|
+
const command = options.codec.deserialize(message.subarray(nameEnd + 1 + 8));
|
|
160
|
+
// the legacy form has no flags of OP_MSG's: nothing is held or repeated for it
|
|
161
|
+
const legacy = await options.command(session, namespace.slice(0, -'.$cmd'.length), command, { exhaustAllowed: false });
|
|
162
|
+
const reply = options.codec.serialize(isTimed(legacy) ? legacy.reply : legacy);
|
|
163
|
+
// OP_REPLY: responseFlags (AwaitCapable), cursorID 0, startingFrom 0, numberReturned 1, then the document
|
|
164
|
+
const fixed = new Uint8Array(20), f = view(fixed);
|
|
165
|
+
f.setInt32(0, 8, true);
|
|
166
|
+
f.setInt32(16, 1, true);
|
|
167
|
+
sink.write(concat([header(HEADER + 20 + reply.length, ++nextResponse, requestId, MongoOp.REPLY), fixed, reply]));
|
|
168
|
+
return;
|
|
169
|
+
}
|
|
170
|
+
throw new Error(`opcode ${opcode} is not served`);
|
|
171
|
+
};
|
|
172
|
+
// One message, in arrival order: parked while a reply is held, answered otherwise.
|
|
173
|
+
const take = (message) => {
|
|
174
|
+
chain = chain.then(() => (closed ? undefined : held !== undefined ? void parked.push(message) : answer(message))).catch((error) => end(error instanceof Error ? error.message : String(error)));
|
|
175
|
+
};
|
|
176
|
+
return {
|
|
177
|
+
data(chunk) {
|
|
178
|
+
if (closed)
|
|
179
|
+
return;
|
|
180
|
+
buffer = buffer.length ? concat([buffer, chunk]) : chunk;
|
|
181
|
+
while (buffer.length >= 4) {
|
|
182
|
+
const length = view(buffer).getInt32(0, true);
|
|
183
|
+
// where the citation stops: what the vendor's server does with a malformed or oversized message is not
|
|
184
|
+
// read; this ends the connection, which no driver's well-formed message reaches
|
|
185
|
+
if (length < HEADER || length > limit) {
|
|
186
|
+
end(`a message of ${length} bytes (accepted: ${HEADER} to ${limit})`);
|
|
187
|
+
return;
|
|
188
|
+
}
|
|
189
|
+
if (buffer.length < length)
|
|
190
|
+
break;
|
|
191
|
+
const message = buffer.subarray(0, length);
|
|
192
|
+
buffer = buffer.subarray(length);
|
|
193
|
+
take(message.slice());
|
|
194
|
+
}
|
|
195
|
+
},
|
|
196
|
+
settled: () => chain,
|
|
197
|
+
close: () => { if (!closed) {
|
|
198
|
+
closed = true;
|
|
199
|
+
release();
|
|
200
|
+
options.close?.(session);
|
|
201
|
+
} },
|
|
202
|
+
};
|
|
203
|
+
};
|
|
204
|
+
}
|
|
205
|
+
/** The bytes of a C string, for a client's OP_QUERY. */
|
|
206
|
+
export const cstring = (text) => concat([enc.encode(text), new Uint8Array(1)]);
|
package/dist/src/observe.js
CHANGED
package/dist/src/pack-fetch.d.ts
CHANGED
|
@@ -5,7 +5,7 @@ import type { ApplicationStandIn } from './events.js';
|
|
|
5
5
|
import { type SocketEngine } from './sockets.js';
|
|
6
6
|
import type { LiveEngine } from './live/types.js';
|
|
7
7
|
import type { WebSocketUpgrade } from './serve-http.js';
|
|
8
|
-
import { type GraphqlPart } from './graphql-wire.js';
|
|
8
|
+
import { type GraphqlPart, type GraphqlLibraryLoader } from './graphql-wire.js';
|
|
9
9
|
/** What a pack declares and writes, and nothing it assembles itself. */
|
|
10
10
|
export type PackParts = {
|
|
11
11
|
manifest: DerivedManifest;
|
|
@@ -39,6 +39,7 @@ export type PackParts = {
|
|
|
39
39
|
* state, served at the manifest's `graphql.paths` (graphql-wire.ts). */
|
|
40
40
|
graphql?: GraphqlPart<HandlerContext> & {
|
|
41
41
|
sdl: string;
|
|
42
|
+
loadLibrary?: GraphqlLibraryLoader;
|
|
42
43
|
};
|
|
43
44
|
/** `semantics/sockets.ts`: each socket's engine, by the id the manifest's `sockets` gives it (sockets.ts). */
|
|
44
45
|
sockets?: Record<string, SocketEngine<HandlerContext>>;
|