@happyvertical/smrt-app-shell 0.0.0-stage → 0.60.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +99 -0
- package/CLAUDE.md +1 -0
- package/LICENSE +7 -0
- package/README.md +173 -2
- package/agents/engine.md +85 -0
- package/dist/engine/app-runtime.d.ts +138 -0
- package/dist/engine/app-runtime.js +545 -0
- package/dist/engine/app-worker.d.ts +1 -0
- package/dist/engine/app-worker.js +62 -0
- package/dist/engine/commands.d.ts +94 -0
- package/dist/engine/commands.js +30 -0
- package/dist/engine/definition.d.ts +192 -0
- package/dist/engine/definition.js +497 -0
- package/dist/engine/errors.d.ts +23 -0
- package/dist/engine/errors.js +35 -0
- package/dist/engine/host.d.ts +153 -0
- package/dist/engine/host.js +791 -0
- package/dist/engine/index.d.ts +6 -0
- package/dist/engine/index.js +7 -0
- package/dist/engine/sandbox.d.ts +60 -0
- package/dist/engine/sandbox.js +260 -0
- package/dist/engine/snapshots.d.ts +7 -0
- package/dist/engine/snapshots.js +54 -0
- package/dist/engine/store.d.ts +39 -0
- package/dist/engine/store.js +32 -0
- package/dist/engine/worker-handle.d.ts +29 -0
- package/dist/engine/worker-handle.js +89 -0
- package/dist/engine/worker-protocol.d.ts +22 -0
- package/dist/engine/worker-protocol.js +20 -0
- package/dist/lib/objects/App.d.ts +60 -0
- package/dist/lib/objects/App.js +273 -0
- package/dist/lib/objects/AppVersion.d.ts +22 -0
- package/dist/lib/objects/AppVersion.js +62 -0
- package/dist/lib/objects/Approval.d.ts +39 -0
- package/dist/lib/objects/Approval.js +101 -0
- package/dist/lib/objects/AssistantMessage.d.ts +24 -0
- package/dist/lib/objects/AssistantMessage.js +63 -0
- package/dist/lib/objects/AssistantThread.d.ts +16 -0
- package/dist/lib/objects/AssistantThread.js +44 -0
- package/dist/lib/objects/index.d.ts +5 -0
- package/dist/lib/objects/index.js +5 -0
- package/dist/lib/server/approvals.d.ts +69 -0
- package/dist/lib/server/approvals.js +239 -0
- package/dist/lib/server/builder.d.ts +16 -0
- package/dist/lib/server/builder.js +71 -0
- package/dist/lib/server/events.d.ts +16 -0
- package/dist/lib/server/events.js +14 -0
- package/dist/lib/server/host.d.ts +22 -0
- package/dist/lib/server/host.js +96 -0
- package/dist/lib/server/owner.d.ts +33 -0
- package/dist/lib/server/owner.js +92 -0
- package/dist/lib/server/version-store.d.ts +20 -0
- package/dist/lib/server/version-store.js +115 -0
- package/dist/manifest.json +1680 -0
- package/dist/smrt-knowledge.okf.md +95 -0
- package/package.json +80 -3
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
export { type Command, type CommandType, type ExecuteOptions, isRiskyCommand, isRiskyTool, type Plan, RISKY_COMMANDS, type Template, } from './commands.ts';
|
|
2
|
+
export { type AppDefinition, builtInFields, coerceValue, convertValue, type DescribedModel, describeApp, EMPTY_APP, FIELD_NAME_PATTERN, FIELD_TYPES, type FieldDef, type FieldType, FORMAT, isValidFieldName, isValidModelName, type MethodDef, type MethodParamDef, type Migration, type ModelDef, RECIPES, type Recipe, type RecipeModel, refuseChanges, type Step, snake, suggestFieldName, tableFor, USER_PKG, validate, } from './definition.ts';
|
|
3
|
+
export { AppError, ApprovalRequiredError, EngineError, type EngineErrorCode, WorkerCrashedError, } from './errors.ts';
|
|
4
|
+
export { AppHost, type ChangeResult, type CleanUpResult, type ExecuteResult, type HostOptions, type MethodResult, type RestResult, type ToolResult, type VersionSummary, } from './host.ts';
|
|
5
|
+
export { checkSyntax, type MethodInput, type MethodOutput, runMethod, SANDBOX_LIMITS, SandboxError, type SandboxLimits, validateWrites, type WritableModel, type Write, } from './sandbox.ts';
|
|
6
|
+
export { MemoryVersionStore, type VersionMigration, type VersionRecord, type VersionStore, } from './store.ts';
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
// The app-shell engine: runs a person's own smrt app from a definition.
|
|
2
|
+
export { isRiskyCommand, isRiskyTool, RISKY_COMMANDS, } from "./commands.js";
|
|
3
|
+
export { builtInFields, coerceValue, convertValue, describeApp, EMPTY_APP, FIELD_NAME_PATTERN, FIELD_TYPES, FORMAT, isValidFieldName, isValidModelName, RECIPES, refuseChanges, snake, suggestFieldName, tableFor, USER_PKG, validate, } from "./definition.js";
|
|
4
|
+
export { AppError, ApprovalRequiredError, EngineError, WorkerCrashedError, } from "./errors.js";
|
|
5
|
+
export { AppHost, } from "./host.js";
|
|
6
|
+
export { checkSyntax, runMethod, SANDBOX_LIMITS, SandboxError, validateWrites, } from "./sandbox.js";
|
|
7
|
+
export { MemoryVersionStore, } from "./store.js";
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import { type FieldDef } from './definition.ts';
|
|
2
|
+
import { EngineError } from './errors.ts';
|
|
3
|
+
export interface MethodInput {
|
|
4
|
+
record: Record<string, unknown> | null;
|
|
5
|
+
params: Record<string, unknown>;
|
|
6
|
+
data: Record<string, Record<string, unknown>[]>;
|
|
7
|
+
}
|
|
8
|
+
export interface Write {
|
|
9
|
+
op: 'create' | 'update' | 'delete';
|
|
10
|
+
model: string;
|
|
11
|
+
id?: string;
|
|
12
|
+
data?: Record<string, unknown>;
|
|
13
|
+
}
|
|
14
|
+
export interface MethodOutput {
|
|
15
|
+
result: unknown;
|
|
16
|
+
writes: Write[];
|
|
17
|
+
}
|
|
18
|
+
export interface SandboxLimits {
|
|
19
|
+
timeoutMs: number;
|
|
20
|
+
memoryBytes: number;
|
|
21
|
+
stackBytes: number;
|
|
22
|
+
maxWrites: number;
|
|
23
|
+
/** Largest JSON the method may return, result and writes together. */
|
|
24
|
+
maxOutputBytes: number;
|
|
25
|
+
/** Largest JSON of data a method may be handed. */
|
|
26
|
+
maxInputBytes: number;
|
|
27
|
+
}
|
|
28
|
+
export declare const SANDBOX_LIMITS: SandboxLimits;
|
|
29
|
+
/** A method failed or broke a limit. */
|
|
30
|
+
export declare class SandboxError extends EngineError {
|
|
31
|
+
constructor(message: string);
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* The WebAssembly heap every method runs in, capped at 128 MB (64 KiB pages).
|
|
35
|
+
*
|
|
36
|
+
* `setMemoryLimit` does not bound large single allocations here: a loop of
|
|
37
|
+
* `'x'.repeat(1e6)` ran to the 2 GB WebAssembly maximum (2,141 allocations,
|
|
38
|
+
* 3.3 s locally, 17 s or more on a CI runner) whatever limit was set, and the
|
|
39
|
+
* interrupt handler is polled only every ~10,000 operations, so the deadline
|
|
40
|
+
* did not fire in that loop either. A hard cap on the heap itself stops every
|
|
41
|
+
* allocation pattern in a fraction of a second; the runtime limit still
|
|
42
|
+
* covers many small allocations.
|
|
43
|
+
*/
|
|
44
|
+
export declare const SANDBOX_HEAP_PAGES: {
|
|
45
|
+
readonly initial: 256;
|
|
46
|
+
readonly maximum: 2048;
|
|
47
|
+
};
|
|
48
|
+
/** Compile-check only: does the method body parse? Returns the problem, or null. */
|
|
49
|
+
export declare function checkSyntax(code: string): Promise<string | null>;
|
|
50
|
+
export declare function runMethod(code: string, input: MethodInput, limits?: Partial<SandboxLimits>): Promise<MethodOutput>;
|
|
51
|
+
export interface WritableModel {
|
|
52
|
+
/** Fields a write may set: visible fields, built-in ones included. */
|
|
53
|
+
fields: Record<string, FieldDef>;
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Check the writes a method proposed before anything is applied: bounded in
|
|
57
|
+
* number, known operations, known models, known fields, values of the right
|
|
58
|
+
* type. Returns clean copies; throws on the first problem.
|
|
59
|
+
*/
|
|
60
|
+
export declare function validateWrites(writes: unknown, models: Record<string, WritableModel>, maxWrites?: number): Write[];
|
|
@@ -0,0 +1,260 @@
|
|
|
1
|
+
// Code methods run in QuickJS (WebAssembly): no filesystem, network, timers or
|
|
2
|
+
// Node APIs. A method is a pure function over a copy of the data. It returns
|
|
3
|
+
// { result, writes } and the engine decides whether to apply the writes.
|
|
4
|
+
//
|
|
5
|
+
// Everything crossing the boundary is JSON text: the input goes in as one
|
|
6
|
+
// string parsed inside the VM, the output comes out as one string parsed
|
|
7
|
+
// here. No host object, function or handle is ever visible to method code.
|
|
8
|
+
import { newQuickJSWASMModuleFromVariant, newVariant, RELEASE_SYNC, shouldInterruptAfterDeadline, } from 'quickjs-emscripten';
|
|
9
|
+
import { coerceValue } from "./definition.js";
|
|
10
|
+
import { EngineError } from "./errors.js";
|
|
11
|
+
export const SANDBOX_LIMITS = {
|
|
12
|
+
timeoutMs: 500,
|
|
13
|
+
memoryBytes: 64 * 1024 * 1024,
|
|
14
|
+
// Small on purpose: past what the host's own stack can hold, V8 throws
|
|
15
|
+
// out of WebAssembly and QuickJS cannot recover (see AGENTS.md).
|
|
16
|
+
stackBytes: 128 * 1024,
|
|
17
|
+
maxWrites: 1000,
|
|
18
|
+
maxOutputBytes: 1024 * 1024,
|
|
19
|
+
maxInputBytes: 8 * 1024 * 1024,
|
|
20
|
+
};
|
|
21
|
+
/** A method failed or broke a limit. */
|
|
22
|
+
export class SandboxError extends EngineError {
|
|
23
|
+
constructor(message) {
|
|
24
|
+
super(message, 'refused');
|
|
25
|
+
this.name = 'SandboxError';
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* The WebAssembly heap every method runs in, capped at 128 MB (64 KiB pages).
|
|
30
|
+
*
|
|
31
|
+
* `setMemoryLimit` does not bound large single allocations here: a loop of
|
|
32
|
+
* `'x'.repeat(1e6)` ran to the 2 GB WebAssembly maximum (2,141 allocations,
|
|
33
|
+
* 3.3 s locally, 17 s or more on a CI runner) whatever limit was set, and the
|
|
34
|
+
* interrupt handler is polled only every ~10,000 operations, so the deadline
|
|
35
|
+
* did not fire in that loop either. A hard cap on the heap itself stops every
|
|
36
|
+
* allocation pattern in a fraction of a second; the runtime limit still
|
|
37
|
+
* covers many small allocations.
|
|
38
|
+
*/
|
|
39
|
+
export const SANDBOX_HEAP_PAGES = { initial: 256, maximum: 2048 };
|
|
40
|
+
let qjs;
|
|
41
|
+
const quickjs = () => {
|
|
42
|
+
qjs ??= newQuickJSWASMModuleFromVariant(newVariant(RELEASE_SYNC, {
|
|
43
|
+
wasmMemory: new WebAssembly.Memory({ ...SANDBOX_HEAP_PAGES }),
|
|
44
|
+
}));
|
|
45
|
+
return qjs;
|
|
46
|
+
};
|
|
47
|
+
/**
|
|
48
|
+
* The host (not the method) threw while the VM was running, e.g. V8 ran out of
|
|
49
|
+
* stack inside WebAssembly. That module instance is no longer trustworthy:
|
|
50
|
+
* drop it so the next method gets a fresh one.
|
|
51
|
+
*/
|
|
52
|
+
const poison = () => {
|
|
53
|
+
qjs = undefined;
|
|
54
|
+
};
|
|
55
|
+
/** Disposing after a host-level failure can abort; that is the same poison. */
|
|
56
|
+
function disposeAll(vm, runtime) {
|
|
57
|
+
try {
|
|
58
|
+
vm.dispose();
|
|
59
|
+
runtime.dispose();
|
|
60
|
+
}
|
|
61
|
+
catch {
|
|
62
|
+
poison();
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
const describeThrown = (e) => typeof e === 'object' && e
|
|
66
|
+
? `${e.name ?? 'Error'}: ${e.message ?? ''}`
|
|
67
|
+
: String(e);
|
|
68
|
+
/** The body is compiled as a function body on its own, so it cannot close the wrapper. */
|
|
69
|
+
const COMPILE = `new Function('record', 'params', 'data', 'app', "'use strict';\\n" + globalThis.__code)`;
|
|
70
|
+
/**
|
|
71
|
+
* Does `code` parse as a function body, and nothing more? V8 parses the body
|
|
72
|
+
* on its own (it never runs), so a body such as `}); evil(); (function(){`
|
|
73
|
+
* cannot close the wrapper: QuickJS's own Function constructor would accept it.
|
|
74
|
+
*/
|
|
75
|
+
function parsesAsBody(code) {
|
|
76
|
+
try {
|
|
77
|
+
// Constructed, never called: parse only.
|
|
78
|
+
new Function('record', 'params', 'data', 'app', `'use strict';\n${code}`);
|
|
79
|
+
return null;
|
|
80
|
+
}
|
|
81
|
+
catch (e) {
|
|
82
|
+
return describeThrown(e);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
/** Compile-check only: does the method body parse? Returns the problem, or null. */
|
|
86
|
+
export async function checkSyntax(code) {
|
|
87
|
+
const structural = parsesAsBody(code);
|
|
88
|
+
if (structural)
|
|
89
|
+
return structural;
|
|
90
|
+
const runtime = (await quickjs()).newRuntime();
|
|
91
|
+
runtime.setMemoryLimit(SANDBOX_LIMITS.memoryBytes);
|
|
92
|
+
runtime.setMaxStackSize(SANDBOX_LIMITS.stackBytes);
|
|
93
|
+
runtime.setInterruptHandler(shouldInterruptAfterDeadline(Date.now() + SANDBOX_LIMITS.timeoutMs));
|
|
94
|
+
const vm = runtime.newContext();
|
|
95
|
+
try {
|
|
96
|
+
const src = vm.newString(code);
|
|
97
|
+
vm.setProp(vm.global, '__code', src);
|
|
98
|
+
src.dispose();
|
|
99
|
+
const r = vm.evalCode(COMPILE);
|
|
100
|
+
if (r.error) {
|
|
101
|
+
const e = vm.dump(r.error);
|
|
102
|
+
r.error.dispose();
|
|
103
|
+
return describeThrown(e);
|
|
104
|
+
}
|
|
105
|
+
r.value.dispose();
|
|
106
|
+
return null;
|
|
107
|
+
}
|
|
108
|
+
finally {
|
|
109
|
+
disposeAll(vm, runtime);
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
export async function runMethod(code, input, limits = {}) {
|
|
113
|
+
const lim = { ...SANDBOX_LIMITS, ...limits };
|
|
114
|
+
const structural = parsesAsBody(code);
|
|
115
|
+
if (structural)
|
|
116
|
+
throw new SandboxError(`method does not parse: ${structural}`);
|
|
117
|
+
const json = JSON.stringify(input);
|
|
118
|
+
if (json.length > lim.maxInputBytes)
|
|
119
|
+
throw new SandboxError('there is too much data for a method to read');
|
|
120
|
+
const deadline = Date.now() + lim.timeoutMs;
|
|
121
|
+
const runtime = (await quickjs()).newRuntime();
|
|
122
|
+
runtime.setMemoryLimit(lim.memoryBytes);
|
|
123
|
+
runtime.setMaxStackSize(lim.stackBytes);
|
|
124
|
+
runtime.setInterruptHandler(shouldInterruptAfterDeadline(deadline));
|
|
125
|
+
const vm = runtime.newContext();
|
|
126
|
+
try {
|
|
127
|
+
for (const [k, v] of [
|
|
128
|
+
['__input', json],
|
|
129
|
+
['__code', code],
|
|
130
|
+
]) {
|
|
131
|
+
const h = vm.newString(v);
|
|
132
|
+
vm.setProp(vm.global, k, h);
|
|
133
|
+
h.dispose();
|
|
134
|
+
}
|
|
135
|
+
const src = `'use strict'; (function () {
|
|
136
|
+
const parse = JSON.parse;
|
|
137
|
+
const stringify = JSON.stringify;
|
|
138
|
+
const raw = globalThis.__input;
|
|
139
|
+
const body = globalThis.__code;
|
|
140
|
+
delete globalThis.__input;
|
|
141
|
+
delete globalThis.__code;
|
|
142
|
+
const input = parse(raw);
|
|
143
|
+
const writes = [];
|
|
144
|
+
// Iterative, so a hostile structure cannot overflow the stack here.
|
|
145
|
+
const tooDeep = (root) => {
|
|
146
|
+
const todo = [[root, 0]];
|
|
147
|
+
let seen = 0;
|
|
148
|
+
while (todo.length) {
|
|
149
|
+
const [v, d] = todo.pop();
|
|
150
|
+
if (v === null || typeof v !== 'object') continue;
|
|
151
|
+
if (d > 64 || ++seen > 200000) return true;
|
|
152
|
+
for (const k of Object.keys(v)) todo.push([v[k], d + 1]);
|
|
153
|
+
}
|
|
154
|
+
return false;
|
|
155
|
+
};
|
|
156
|
+
const push = (w) => {
|
|
157
|
+
if (writes.length >= ${lim.maxWrites}) throw new Error('a method may propose at most ${lim.maxWrites} writes');
|
|
158
|
+
if (tooDeep(w)) throw new Error('a write is nested too deeply');
|
|
159
|
+
writes.push(parse(stringify(w)));
|
|
160
|
+
};
|
|
161
|
+
const app = Object.freeze({
|
|
162
|
+
create: (model, data) => { push({ op: 'create', model, data }); },
|
|
163
|
+
update: (model, id, data) => { push({ op: 'update', model, id, data }); },
|
|
164
|
+
delete: (model, id) => { push({ op: 'delete', model, id }); },
|
|
165
|
+
});
|
|
166
|
+
const run = new Function('record', 'params', 'data', 'app', "'use strict';\\n" + body);
|
|
167
|
+
const result = run(input.record, input.params, input.data, app);
|
|
168
|
+
if (result && typeof result.then === 'function')
|
|
169
|
+
throw new Error('a method must return a value, not a promise');
|
|
170
|
+
if (tooDeep(result)) throw new Error('the result is nested too deeply');
|
|
171
|
+
const out = stringify({ result: result === undefined ? null : result, writes });
|
|
172
|
+
if (typeof out !== 'string' || out.length > ${lim.maxOutputBytes})
|
|
173
|
+
throw new Error('the method returned too much');
|
|
174
|
+
return out;
|
|
175
|
+
})()`;
|
|
176
|
+
const r = vm.evalCode(src, 'method.js');
|
|
177
|
+
if (r.error) {
|
|
178
|
+
const e = vm.dump(r.error);
|
|
179
|
+
r.error.dispose();
|
|
180
|
+
const timedOut = Date.now() >= deadline;
|
|
181
|
+
throw new SandboxError(timedOut
|
|
182
|
+
? `method ran too long (limit ${lim.timeoutMs} ms)`
|
|
183
|
+
: `method failed: ${describeThrown(e)}`);
|
|
184
|
+
}
|
|
185
|
+
const text = vm.getString(r.value);
|
|
186
|
+
r.value.dispose();
|
|
187
|
+
let out;
|
|
188
|
+
try {
|
|
189
|
+
out = JSON.parse(text);
|
|
190
|
+
}
|
|
191
|
+
catch {
|
|
192
|
+
throw new SandboxError('method returned something that is not JSON');
|
|
193
|
+
}
|
|
194
|
+
const o = out;
|
|
195
|
+
if (!o || typeof o !== 'object' || !Array.isArray(o.writes))
|
|
196
|
+
throw new SandboxError('method returned an unexpected shape');
|
|
197
|
+
return { result: o.result ?? null, writes: o.writes };
|
|
198
|
+
}
|
|
199
|
+
catch (e) {
|
|
200
|
+
if (e instanceof SandboxError)
|
|
201
|
+
throw e;
|
|
202
|
+
poison();
|
|
203
|
+
throw new SandboxError(`method crashed the sandbox: ${e.message}`);
|
|
204
|
+
}
|
|
205
|
+
finally {
|
|
206
|
+
disposeAll(vm, runtime);
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
const isPlain = (v) => !!v && typeof v === 'object' && !Array.isArray(v);
|
|
210
|
+
/**
|
|
211
|
+
* Check the writes a method proposed before anything is applied: bounded in
|
|
212
|
+
* number, known operations, known models, known fields, values of the right
|
|
213
|
+
* type. Returns clean copies; throws on the first problem.
|
|
214
|
+
*/
|
|
215
|
+
export function validateWrites(writes, models, maxWrites = SANDBOX_LIMITS.maxWrites) {
|
|
216
|
+
if (!Array.isArray(writes))
|
|
217
|
+
throw new SandboxError('writes must be a list');
|
|
218
|
+
if (writes.length > maxWrites)
|
|
219
|
+
throw new SandboxError(`too many writes (limit ${maxWrites})`);
|
|
220
|
+
const out = [];
|
|
221
|
+
writes.forEach((w, i) => {
|
|
222
|
+
const at = `write ${i + 1}`;
|
|
223
|
+
if (!isPlain(w))
|
|
224
|
+
throw new SandboxError(`${at}: not an object`);
|
|
225
|
+
for (const k of Object.keys(w))
|
|
226
|
+
if (!['op', 'model', 'id', 'data'].includes(k))
|
|
227
|
+
throw new SandboxError(`${at}: unknown key '${k}'`);
|
|
228
|
+
if (w.op !== 'create' && w.op !== 'update' && w.op !== 'delete')
|
|
229
|
+
throw new SandboxError(`${at}: unknown operation '${String(w.op)}'`);
|
|
230
|
+
if (typeof w.model !== 'string' || !Object.hasOwn(models, w.model))
|
|
231
|
+
throw new SandboxError(`${at}: no model '${String(w.model)}'`);
|
|
232
|
+
const clean = { op: w.op, model: w.model };
|
|
233
|
+
if (w.op !== 'create') {
|
|
234
|
+
if (typeof w.id !== 'string' || !w.id)
|
|
235
|
+
throw new SandboxError(`${at}: ${w.op} needs a record id`);
|
|
236
|
+
clean.id = w.id;
|
|
237
|
+
}
|
|
238
|
+
if (w.op === 'delete') {
|
|
239
|
+
if (w.data !== undefined)
|
|
240
|
+
throw new SandboxError(`${at}: delete takes no data`);
|
|
241
|
+
}
|
|
242
|
+
else {
|
|
243
|
+
if (!isPlain(w.data))
|
|
244
|
+
throw new SandboxError(`${at}: ${w.op} needs data`);
|
|
245
|
+
const fields = models[w.model].fields;
|
|
246
|
+
const data = {};
|
|
247
|
+
for (const [k, v] of Object.entries(w.data)) {
|
|
248
|
+
if (!Object.hasOwn(fields, k))
|
|
249
|
+
throw new SandboxError(`${at}: ${w.model} has no field '${k}'`);
|
|
250
|
+
const c = coerceValue(fields[k], v);
|
|
251
|
+
if (!c.ok)
|
|
252
|
+
throw new SandboxError(`${at}: ${w.model}.${k} ${c.error}`);
|
|
253
|
+
data[k] = v;
|
|
254
|
+
}
|
|
255
|
+
clean.data = data;
|
|
256
|
+
}
|
|
257
|
+
out.push(clean);
|
|
258
|
+
});
|
|
259
|
+
return out;
|
|
260
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/** How many `method-*` snapshots stay (each applied method write takes one). */
|
|
2
|
+
export declare const METHOD_SNAPSHOTS_KEPT = 20;
|
|
3
|
+
export type SnapshotKind = 'method' | 'migrate' | 'schema';
|
|
4
|
+
/** Keep the newest `keep` `method-*` snapshots; delete the rest. */
|
|
5
|
+
export declare function pruneMethodSnapshots(dir: string, keep?: number): number;
|
|
6
|
+
/** Delete `migrate-*` and `schema-*` snapshots taken before `beforeMs`. */
|
|
7
|
+
export declare function pruneMigrationSnapshots(dir: string, beforeMs: number): number;
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
// Retention for `<dataDir>/snapshots`. Only plain files whose names match the
|
|
2
|
+
// engine's own pattern are ever touched: never a directory, a symlink, or a
|
|
3
|
+
// file someone else put there.
|
|
4
|
+
import { readdirSync, unlinkSync } from 'node:fs';
|
|
5
|
+
import { join } from 'node:path';
|
|
6
|
+
/** How many `method-*` snapshots stay (each applied method write takes one). */
|
|
7
|
+
export const METHOD_SNAPSHOTS_KEPT = 20;
|
|
8
|
+
/** `<epoch ms>-<counter>-<kind...>.db`, as `AppRuntime.snapshot` names them. */
|
|
9
|
+
const NAME = /^(\d{1,16})-(\d{1,9})-(method-|migrate-|schema)[A-Za-z0-9._-]*\.db$/;
|
|
10
|
+
function ours(dir) {
|
|
11
|
+
let entries;
|
|
12
|
+
try {
|
|
13
|
+
entries = readdirSync(dir, { withFileTypes: true });
|
|
14
|
+
}
|
|
15
|
+
catch {
|
|
16
|
+
return [];
|
|
17
|
+
}
|
|
18
|
+
const out = [];
|
|
19
|
+
for (const e of entries) {
|
|
20
|
+
if (!e.isFile())
|
|
21
|
+
continue; // symlinks and directories are left alone
|
|
22
|
+
const m = NAME.exec(e.name);
|
|
23
|
+
if (!m)
|
|
24
|
+
continue;
|
|
25
|
+
out.push({
|
|
26
|
+
name: e.name,
|
|
27
|
+
at: Number(m[1]),
|
|
28
|
+
n: Number(m[2]),
|
|
29
|
+
kind: m[3].replace(/-$/, ''),
|
|
30
|
+
});
|
|
31
|
+
}
|
|
32
|
+
return out.sort((a, b) => a.at - b.at || a.n - b.n);
|
|
33
|
+
}
|
|
34
|
+
function remove(dir, files) {
|
|
35
|
+
let removed = 0;
|
|
36
|
+
for (const f of files)
|
|
37
|
+
try {
|
|
38
|
+
unlinkSync(join(dir, f.name));
|
|
39
|
+
removed++;
|
|
40
|
+
}
|
|
41
|
+
catch {
|
|
42
|
+
// already gone or not ours to remove: retention is best effort
|
|
43
|
+
}
|
|
44
|
+
return removed;
|
|
45
|
+
}
|
|
46
|
+
/** Keep the newest `keep` `method-*` snapshots; delete the rest. */
|
|
47
|
+
export function pruneMethodSnapshots(dir, keep = METHOD_SNAPSHOTS_KEPT) {
|
|
48
|
+
const method = ours(dir).filter((f) => f.kind === 'method');
|
|
49
|
+
return remove(dir, method.slice(0, Math.max(0, method.length - keep)));
|
|
50
|
+
}
|
|
51
|
+
/** Delete `migrate-*` and `schema-*` snapshots taken before `beforeMs`. */
|
|
52
|
+
export function pruneMigrationSnapshots(dir, beforeMs) {
|
|
53
|
+
return remove(dir, ours(dir).filter((f) => f.kind !== 'method' && f.at < beforeMs));
|
|
54
|
+
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import type { AppDefinition, Step } from './definition.ts';
|
|
2
|
+
export interface VersionMigration {
|
|
3
|
+
steps: Step[];
|
|
4
|
+
/** Archived data was cleaned up: this change can no longer be undone. */
|
|
5
|
+
purged?: boolean;
|
|
6
|
+
}
|
|
7
|
+
export interface VersionRecord {
|
|
8
|
+
version: number;
|
|
9
|
+
parent: number | null;
|
|
10
|
+
author: string;
|
|
11
|
+
summary: string;
|
|
12
|
+
definition: AppDefinition;
|
|
13
|
+
migration: VersionMigration | null;
|
|
14
|
+
createdAt: string;
|
|
15
|
+
}
|
|
16
|
+
export interface VersionStore {
|
|
17
|
+
/** The current version, or null for a store that has never held one. */
|
|
18
|
+
current(): Promise<number | null>;
|
|
19
|
+
setCurrent(version: number): Promise<void>;
|
|
20
|
+
get(version: number): Promise<VersionRecord | null>;
|
|
21
|
+
/** Every version, oldest first. */
|
|
22
|
+
list(): Promise<VersionRecord[]>;
|
|
23
|
+
append(record: VersionRecord): Promise<void>;
|
|
24
|
+
/** Mark a version's migration `purged`. Never delete the record. */
|
|
25
|
+
markPurged(version: number): Promise<void>;
|
|
26
|
+
/** Highest version number held, or -1 when empty. */
|
|
27
|
+
maxVersion(): Promise<number>;
|
|
28
|
+
}
|
|
29
|
+
export declare class MemoryVersionStore implements VersionStore {
|
|
30
|
+
private cur;
|
|
31
|
+
private records;
|
|
32
|
+
current(): Promise<number | null>;
|
|
33
|
+
setCurrent(version: number): Promise<void>;
|
|
34
|
+
get(version: number): Promise<VersionRecord | null>;
|
|
35
|
+
list(): Promise<VersionRecord[]>;
|
|
36
|
+
append(record: VersionRecord): Promise<void>;
|
|
37
|
+
markPurged(version: number): Promise<void>;
|
|
38
|
+
maxVersion(): Promise<number>;
|
|
39
|
+
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
export class MemoryVersionStore {
|
|
2
|
+
cur = null;
|
|
3
|
+
records = new Map();
|
|
4
|
+
async current() {
|
|
5
|
+
return this.cur;
|
|
6
|
+
}
|
|
7
|
+
async setCurrent(version) {
|
|
8
|
+
this.cur = version;
|
|
9
|
+
}
|
|
10
|
+
async get(version) {
|
|
11
|
+
const r = this.records.get(version);
|
|
12
|
+
return r ? structuredClone(r) : null;
|
|
13
|
+
}
|
|
14
|
+
async list() {
|
|
15
|
+
return [...this.records.values()]
|
|
16
|
+
.sort((a, b) => a.version - b.version)
|
|
17
|
+
.map((r) => structuredClone(r));
|
|
18
|
+
}
|
|
19
|
+
async append(record) {
|
|
20
|
+
if (this.records.has(record.version))
|
|
21
|
+
throw new Error(`version ${record.version} already exists`);
|
|
22
|
+
this.records.set(record.version, structuredClone(record));
|
|
23
|
+
}
|
|
24
|
+
async markPurged(version) {
|
|
25
|
+
const r = this.records.get(version);
|
|
26
|
+
if (r?.migration)
|
|
27
|
+
r.migration = { ...r.migration, purged: true };
|
|
28
|
+
}
|
|
29
|
+
async maxVersion() {
|
|
30
|
+
return this.records.size ? Math.max(...this.records.keys()) : -1;
|
|
31
|
+
}
|
|
32
|
+
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import type { AppDefinition, Migration } from './definition.ts';
|
|
2
|
+
import type { BootInfo, WorkerOp } from './worker-protocol.ts';
|
|
3
|
+
/** The app answered with an error (as opposed to the thread dying). */
|
|
4
|
+
export declare class AppCallError extends Error {
|
|
5
|
+
readonly user: boolean;
|
|
6
|
+
constructor(message: string, user: boolean);
|
|
7
|
+
}
|
|
8
|
+
/** The app's heap is bounded; a runaway app cannot starve the host. */
|
|
9
|
+
export declare const WORKER_HEAP_MB = 512;
|
|
10
|
+
export declare class AppWorker {
|
|
11
|
+
readonly version: number;
|
|
12
|
+
private worker;
|
|
13
|
+
private next;
|
|
14
|
+
private dead;
|
|
15
|
+
private pending;
|
|
16
|
+
readonly ready: Promise<BootInfo>;
|
|
17
|
+
constructor(workerUrl: URL | string, init: {
|
|
18
|
+
dataDir: string;
|
|
19
|
+
def: AppDefinition;
|
|
20
|
+
version: number;
|
|
21
|
+
migration?: Migration;
|
|
22
|
+
});
|
|
23
|
+
get alive(): boolean;
|
|
24
|
+
private settle;
|
|
25
|
+
call<T = unknown>(op: WorkerOp, ...args: unknown[]): Promise<T>;
|
|
26
|
+
/** Let in-flight calls finish (up to `ms`), then stop the thread. */
|
|
27
|
+
retire(ms?: number): Promise<void>;
|
|
28
|
+
terminate(): Promise<number>;
|
|
29
|
+
}
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
// The host's handle on one app runtime thread.
|
|
2
|
+
import { Worker } from 'node:worker_threads';
|
|
3
|
+
import { EngineError, WorkerCrashedError } from "./errors.js";
|
|
4
|
+
/** The app answered with an error (as opposed to the thread dying). */
|
|
5
|
+
export class AppCallError extends Error {
|
|
6
|
+
user;
|
|
7
|
+
constructor(message, user) {
|
|
8
|
+
super(message);
|
|
9
|
+
this.name = 'AppCallError';
|
|
10
|
+
this.user = user;
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
/** The app's heap is bounded; a runaway app cannot starve the host. */
|
|
14
|
+
export const WORKER_HEAP_MB = 512;
|
|
15
|
+
export class AppWorker {
|
|
16
|
+
version;
|
|
17
|
+
worker;
|
|
18
|
+
next = 1;
|
|
19
|
+
dead = null;
|
|
20
|
+
pending = new Map();
|
|
21
|
+
ready;
|
|
22
|
+
constructor(workerUrl, init) {
|
|
23
|
+
this.version = init.version;
|
|
24
|
+
this.worker = new Worker(workerUrl, {
|
|
25
|
+
workerData: { appWorker: true, ...init },
|
|
26
|
+
resourceLimits: { maxOldGenerationSizeMb: WORKER_HEAP_MB },
|
|
27
|
+
});
|
|
28
|
+
const die = (e) => {
|
|
29
|
+
this.dead ??= e;
|
|
30
|
+
for (const p of this.pending.values())
|
|
31
|
+
p.reject(this.dead);
|
|
32
|
+
this.pending.clear();
|
|
33
|
+
};
|
|
34
|
+
this.ready = new Promise((resolve, reject) => {
|
|
35
|
+
this.worker.once('message', (m) => {
|
|
36
|
+
if (m.ready)
|
|
37
|
+
resolve(m);
|
|
38
|
+
else
|
|
39
|
+
reject(new EngineError(m.error ?? 'the app did not start', 'migration_failed'));
|
|
40
|
+
this.worker.on('message', (r) => this.settle(r));
|
|
41
|
+
});
|
|
42
|
+
this.worker.once('error', (e) => {
|
|
43
|
+
die(new WorkerCrashedError(`the app stopped: ${e.message}`));
|
|
44
|
+
reject(e);
|
|
45
|
+
});
|
|
46
|
+
this.worker.once('exit', () => {
|
|
47
|
+
die(new WorkerCrashedError());
|
|
48
|
+
reject(new WorkerCrashedError('the app stopped before it was ready'));
|
|
49
|
+
});
|
|
50
|
+
});
|
|
51
|
+
// A rejected `ready` nobody awaited (a worker that died later) is not an error.
|
|
52
|
+
this.ready.catch(() => { });
|
|
53
|
+
}
|
|
54
|
+
get alive() {
|
|
55
|
+
return !this.dead;
|
|
56
|
+
}
|
|
57
|
+
settle(r) {
|
|
58
|
+
const p = this.pending.get(r.id);
|
|
59
|
+
if (!p)
|
|
60
|
+
return;
|
|
61
|
+
this.pending.delete(r.id);
|
|
62
|
+
if (r.ok)
|
|
63
|
+
p.resolve(r.value);
|
|
64
|
+
else
|
|
65
|
+
p.reject(new AppCallError(r.error, r.user));
|
|
66
|
+
}
|
|
67
|
+
call(op, ...args) {
|
|
68
|
+
if (this.dead)
|
|
69
|
+
return Promise.reject(this.dead);
|
|
70
|
+
const id = this.next++;
|
|
71
|
+
return new Promise((resolve, reject) => {
|
|
72
|
+
this.pending.set(id, {
|
|
73
|
+
resolve: resolve,
|
|
74
|
+
reject,
|
|
75
|
+
});
|
|
76
|
+
this.worker.postMessage({ id, op, args });
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
/** Let in-flight calls finish (up to `ms`), then stop the thread. */
|
|
80
|
+
async retire(ms = 2000) {
|
|
81
|
+
const until = Date.now() + ms;
|
|
82
|
+
while (this.pending.size && !this.dead && Date.now() < until)
|
|
83
|
+
await new Promise((r) => setTimeout(r, 20));
|
|
84
|
+
await this.worker.terminate();
|
|
85
|
+
}
|
|
86
|
+
terminate() {
|
|
87
|
+
return this.worker.terminate();
|
|
88
|
+
}
|
|
89
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
export declare const WORKER_OPS: readonly ["list", "get", "create", "update", "remove", "runMethod", "tools", "callTool", "rest", "snapshot", "previewConversion", "countValues", "countRows", "countArchives", "countOverwrites", "pruneSnapshots", "purgeArchives", "importRecords"];
|
|
2
|
+
export type WorkerOp = (typeof WORKER_OPS)[number];
|
|
3
|
+
export type WorkerCall = {
|
|
4
|
+
id: number;
|
|
5
|
+
op: string;
|
|
6
|
+
args: unknown[];
|
|
7
|
+
};
|
|
8
|
+
export type WorkerReply = {
|
|
9
|
+
id: number;
|
|
10
|
+
ok: true;
|
|
11
|
+
value: unknown;
|
|
12
|
+
} | {
|
|
13
|
+
id: number;
|
|
14
|
+
ok: false;
|
|
15
|
+
error: string;
|
|
16
|
+
user: boolean;
|
|
17
|
+
};
|
|
18
|
+
export interface BootInfo {
|
|
19
|
+
statements: string[];
|
|
20
|
+
snapshot: string | null;
|
|
21
|
+
bootMs: number;
|
|
22
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
export const WORKER_OPS = [
|
|
2
|
+
'list',
|
|
3
|
+
'get',
|
|
4
|
+
'create',
|
|
5
|
+
'update',
|
|
6
|
+
'remove',
|
|
7
|
+
'runMethod',
|
|
8
|
+
'tools',
|
|
9
|
+
'callTool',
|
|
10
|
+
'rest',
|
|
11
|
+
'snapshot',
|
|
12
|
+
'previewConversion',
|
|
13
|
+
'countValues',
|
|
14
|
+
'countRows',
|
|
15
|
+
'countArchives',
|
|
16
|
+
'countOverwrites',
|
|
17
|
+
'pruneSnapshots',
|
|
18
|
+
'purgeArchives',
|
|
19
|
+
'importRecords',
|
|
20
|
+
];
|