flatsql 0.4.3 → 1.0.0

package/dist/stress/workload.js

@@ -0,0 +1,144 @@
+export const DEFAULT_USE_CASES = [
+    {
+        id: 'schema-sweep',
+        label: 'SDS schema sweep',
+        operation: 'schema-parse',
+        description: 'Discover every SDS FlatBuffer schema and record compatibility metadata.',
+    },
+    {
+        id: 'bulk-streaming-ingest',
+        label: 'Bulk streaming ingest',
+        operation: 'stream-ingest',
+        description: 'Stream generated FlatBuffers in size-prefixed batches.',
+    },
+    {
+        id: 'hot-file-id-cache',
+        label: 'Hot FILE_ID query cache',
+        operation: 'query-template',
+        description: 'Repeat the same PNM/DPM-style FILE_ID lookup to measure cache behavior.',
+    },
+    {
+        id: 'cold-file-id-fanout',
+        label: 'Cold FILE_ID fanout',
+        operation: 'query-template',
+        description: 'Fan out mostly unique FILE_ID lookups and measure cache churn.',
+    },
+    {
+        id: 'mixed-hot-cold-query',
+        label: 'Mixed hot/cold query distribution',
+        operation: 'query-template',
+        description: 'Run a skewed query mix with hot and long-tail FILE_ID values.',
+    },
+    {
+        id: 'raw-flatbuffer-retrieval',
+        label: 'Raw FlatBuffer retrieval',
+        operation: 'raw-flatbuffer',
+        description: 'Return original FlatBuffer bytes through indexed lookup paths.',
+    },
+    {
+        id: 'sql-projection-query',
+        label: 'SQL projection query',
+        operation: 'parameterized-sql',
+        description: 'Return a narrow SQL projection to compare against binary payload transfer.',
+    },
+    {
+        id: 'raw-sql-query',
+        label: 'Raw SQL query',
+        operation: 'raw-sql',
+        description: 'Execute untemplated SQL and compare request overhead with query templates.',
+    },
+    {
+        id: 'large-result-query',
+        label: 'Large result query',
+        operation: 'large-result',
+        description: 'Return more rows than the result-cache row cap and measure transport cost.',
+    },
+    {
+        id: 'node-fanout-query',
+        label: 'Node fanout query',
+        operation: 'fanout-query',
+        description: 'Send one logical query across many nodes and measure tail latency.',
+    },
+    {
+        id: 'backfill-sync',
+        label: 'Backfill sync',
+        operation: 'backfill-stream',
+        description: 'Bootstrap a node by streaming records from another source.',
+    },
+    {
+        id: 'restart-rebuild',
+        label: 'Restart and rebuild',
+        operation: 'restart-rebuild',
+        description: 'Restart a node and measure rebuild or reload behavior.',
+    },
+    {
+        id: 'cache-invalidation-churn',
+        label: 'Cache invalidation churn',
+        operation: 'ingest-query-churn',
+        description: 'Alternate ingest and hot queries to expose broad invalidation cost.',
+    },
+    {
+        id: 'bandwidth-constrained-streaming',
+        label: 'Bandwidth-constrained streaming',
+        operation: 'throttled-stream',
+        description: 'Measure chunked streaming behavior under an artificial transport limit.',
+    },
+    {
+        id: 'sds-domain-query-pack',
+        label: 'SDS domain query pack',
+        operation: 'domain-query',
+        description: 'Exercise named space-domain lookups such as catalog, ephemeris, sensor, and weather queries.',
+    },
+    {
+        id: 'udl-style-api-shape',
+        label: 'UDL-style API comparison shape',
+        operation: 'api-shape',
+        description: 'Run centralized API-shaped requests without calling an external UDL service.',
+    },
+];
+export function normalizeStressRunConfig(config = {}) {
+    const mode = config.mode ?? 'smoke';
+    return {
+        mode,
+        runId: config.runId ?? `sds-${mode}-${Date.now()}`,
+        nodeCount: config.nodeCount ?? (mode === 'full' ? 100 : 2),
+        nodeIdOffset: config.nodeIdOffset ?? 0,
+        nodeStorageGb: config.nodeStorageGb ?? (mode === 'full' ? 1 : 0.01),
+        recordsPerNode: config.recordsPerNode ?? (mode === 'full' ? 20_000 : 100),
+        batchBytes: config.batchBytes ?? 1024 * 1024,
+        queryConcurrency: config.queryConcurrency ?? (mode === 'full' ? 64 : 4),
+        hotQueryRatio: config.hotQueryRatio ?? 0.9,
+        runtime: config.runtime ?? 'standalone',
+        transport: config.transport ?? 'in-process',
+        nodeIsolation: config.nodeIsolation ?? 'logical',
+        seed: config.seed ?? 0x5d5,
+    };
+}
+export function buildNodeAssignments(schemas, config) {
+    const schemaNames = schemas.map((schema) => schema.name);
+    return Array.from({ length: config.nodeCount }, (_, localNodeIndex) => {
+        const nodeId = config.nodeIdOffset + localNodeIndex;
+        const assigned = schemaNames.filter((_, index) => index % config.nodeCount === localNodeIndex);
+        if (assigned.length === 0 && schemaNames.length > 0) {
+            assigned.push(schemaNames[nodeId % schemaNames.length]);
+        }
+        return {
+            nodeId,
+            storageGb: config.nodeStorageGb,
+            schemas: assigned,
+        };
+    });
+}
+export function buildWorkloadManifest(config, schemas) {
+    const normalized = normalizeStressRunConfig(config);
+    const sortedSchemas = [...schemas].sort((left, right) => left.name.localeCompare(right.name));
+    return {
+        ...normalized,
+        createdAt: new Date().toISOString(),
+        schemaCount: sortedSchemas.length,
+        schemas: sortedSchemas,
+        useCases: DEFAULT_USE_CASES,
+        assignments: buildNodeAssignments(sortedSchemas, normalized),
+    };
+}
+//# sourceMappingURL=workload.js.map

package/dist/stress/workload.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"workload.js","sourceRoot":"","sources":["../../src/stress/workload.ts"],"names":[],"mappings":"AASA,MAAM,CAAC,MAAM,iBAAiB,GAAoB;IAChD;QACE,EAAE,EAAE,cAAc;QAClB,KAAK,EAAE,kBAAkB;QACzB,SAAS,EAAE,cAAc;QACzB,WAAW,EAAE,yEAAyE;KACvF;IACD;QACE,EAAE,EAAE,uBAAuB;QAC3B,KAAK,EAAE,uBAAuB;QAC9B,SAAS,EAAE,eAAe;QAC1B,WAAW,EAAE,wDAAwD;KACtE;IACD;QACE,EAAE,EAAE,mBAAmB;QACvB,KAAK,EAAE,yBAAyB;QAChC,SAAS,EAAE,gBAAgB;QAC3B,WAAW,EAAE,yEAAyE;KACvF;IACD;QACE,EAAE,EAAE,qBAAqB;QACzB,KAAK,EAAE,qBAAqB;QAC5B,SAAS,EAAE,gBAAgB;QAC3B,WAAW,EAAE,gEAAgE;KAC9E;IACD;QACE,EAAE,EAAE,sBAAsB;QAC1B,KAAK,EAAE,mCAAmC;QAC1C,SAAS,EAAE,gBAAgB;QAC3B,WAAW,EAAE,+DAA+D;KAC7E;IACD;QACE,EAAE,EAAE,0BAA0B;QAC9B,KAAK,EAAE,0BAA0B;QACjC,SAAS,EAAE,gBAAgB;QAC3B,WAAW,EAAE,gEAAgE;KAC9E;IACD;QACE,EAAE,EAAE,sBAAsB;QAC1B,KAAK,EAAE,sBAAsB;QAC7B,SAAS,EAAE,mBAAmB;QAC9B,WAAW,EAAE,4EAA4E;KAC1F;IACD;QACE,EAAE,EAAE,eAAe;QACnB,KAAK,EAAE,eAAe;QACtB,SAAS,EAAE,SAAS;QACpB,WAAW,EAAE,4EAA4E;KAC1F;IACD;QACE,EAAE,EAAE,oBAAoB;QACxB,KAAK,EAAE,oBAAoB;QAC3B,SAAS,EAAE,cAAc;QACzB,WAAW,EAAE,4EAA4E;KAC1F;IACD;QACE,EAAE,EAAE,mBAAmB;QACvB,KAAK,EAAE,mBAAmB;QAC1B,SAAS,EAAE,cAAc;QACzB,WAAW,EAAE,oEAAoE;KAClF;IACD;QACE,EAAE,EAAE,eAAe;QACnB,KAAK,EAAE,eAAe;QACtB,SAAS,EAAE,iBAAiB;QAC5B,WAAW,EAAE,4DAA4D;KAC1E;IACD;QACE,EAAE,EAAE,iBAAiB;QACrB,KAAK,EAAE,qBAAqB;QAC5B,SAAS,EAAE,iBAAiB;QAC5B,WAAW,EAAE,wDAAwD;KACtE;IACD;QACE,EAAE,EAAE,0BAA0B;QAC9B,KAAK,EAAE,0BAA0B;QACjC,SAAS,EAAE,oBAAoB;QAC/B,WAAW,EAAE,qEAAqE;KACnF;IACD;QACE,EAAE,EAAE,iCAAiC;QACrC,KAAK,EAAE,iCAAiC;QACxC,SAAS,EAAE,kBAAkB;QAC7B,WAAW,EAAE,yEAAyE;KACvF;IACD;QACE,EAAE,EAAE,uBAAuB;QAC3B,KAAK,EAAE,uBAAuB;QAC9B,SAAS,EAAE,cAAc;QACzB,WAAW,EAAE,8FAA8F;KAC5G;IACD;QACE,EAAE,EAAE,qBAAqB;QACzB,KAAK,EAAE,gCAAgC;QACvC,SAAS,EAAE,WAAW;QACtB,WAAW,EAAE,8EAA8E;KAC5F;CACF,CAAC;AAEF,MAAM,UAAU,wBAAwB,CAAC,SAA0B,EAAE;IACnE,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,IAAI,OAAO,CAAC;IACpC,OAAO;QACL,IAAI;QACJ,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,OAAO,IAAI,IAAI,IAAI,CAAC,GAAG,EAAE,EAAE;QAClD,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1D,YAAY,EAAE,MAAM,CAAC,YAAY,IAAI,CAAC;QACtC,aAAa,EAAE,MAAM,CAAC,aAAa,IAAI,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACnE,cAAc,EAAE,MAAM,CAAC,cAAc,IAAI,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC;QACzE,UAAU,EAAE,MAAM,CAAC,UAAU,IAAI,IAAI,GAAG,IAAI;QAC5C,gBAAgB,EAAE,MAAM,CAAC,gBAAgB,IAAI,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACvE,aAAa,EAAE,MAAM,CAAC,aAAa,IAAI,GAAG;QAC1C,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,YAAY;QACvC,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,YAAY;QAC3C,aAAa,EAAE,MAAM,CAAC,aAAa,IAAI,SAAS;QAChD,IAAI,EAAE,MAAM,CAAC,IAAI,IAAI,KAAK;KAC3B,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,OAAiC,EACjC,MAAuF;IAEvF,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACzD,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,EAAE,cAAc,EAAE,EAAE;QACpE,MAAM,MAAM,GAAG,MAAM,CAAC,YAAY,GAAG,cAAc,CAAC;QACpD,MAAM,QAAQ,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,MAAM,CAAC,SAAS,KAAK,cAAc,CAAC,CAAC;QAC/F,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACpD,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC;QAC1D,CAAC;QACD,OAAO;YACL,MAAM;YACN,SAAS,EAAE,MAAM,CAAC,aAAa;YAC/B,OAAO,EAAE,QAAQ;SAClB,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,qBAAqB,CACnC,MAAuB,EACvB,OAAiC;IAEjC,MAAM,UAAU,GAAG,wBAAwB,CAAC,MAAM,CAAC,CAAC;IACpD,MAAM,aAAa,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;IAC9F,OAAO;QACL,GAAG,UAAU;QACb,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,WAAW,EAAE,aAAa,CAAC,MAAM;QACjC,OAAO,EAAE,aAAa;QACtB,QAAQ,EAAE,iBAAiB;QAC3B,WAAW,EAAE,oBAAoB,CAAC,aAAa,EAAE,UAAU,CAAC;KAC7D,CAAC;AACJ,CAAC"}

package/docs/agents/response-artifacts.md

@@ -0,0 +1,65 @@
+# Agent Guide: Response Bandwidth
+
+Use this guide when modifying FlatSQL response caching, SDS stress tests, or Space Data Network FILE_ID behavior.
+
+## Rules
+
+- Do not add FlatSQL-specific fields or annotations to canonical SDS schemas.
+- Treat `FILE_ID` as a Space Data Network publish-event policy key.
+- Use runtime indexes or external SDN index profiles for lookup speed.
+- Measure response bytes separately from query latency and cache hits.
+- Prefer raw FlatBuffer artifacts for repeated high-volume responses.
+- Tune native query-template cache limits with `configureQueryCache({ maxEntries, maxRows })` when hot FILE_ID traffic needs more than the defaults.
+
+## Library Responsibilities
+
+FlatSQL response bandwidth work belongs in these primitives:
+
+- native `buildResponseArtifactCacheKey(...)` bindings for deterministic query/result cache keys
+- `createQueryResponseArtifact(...)` for immutable byte artifacts
+- `getResponseArtifactChunk(...)` for range-addressable chunk reads
+- `MemoryResponseArtifactCache.getOrCreateByKey(...)` for bounded process-local artifact reuse with native keys
+
+Deployment code is responsible for CDN, browser Cache API, WasmEdge host filesystem cache, auth, quotas, routing, and regional replication.
+
+## Required Tests
+
+When changing response artifacts, run:
+
+```bash
+npm test -- --runInBand --runTestsByPath test/response-artifact.test.ts
+npm test -- --runInBand --runTestsByPath test/wasm-query-params.test.ts test/standalone-shim.test.ts test/standalone-artifact-builder.test.ts test/wasmedge-runner.test.ts
+npm test -- --runInBand --runTestsByPath test/stress-harness.test.ts
+npm run build
+```
+
+Before claiming production readiness, also run:
+
+```bash
+npm test -- --runInBand
+npm run stress:sds:full-config -- --run-id sds-full-config-check
+docker compose -f stress/results/sds-full-config-check/docker-compose.full.yml config
+git diff --check
+```
+
+## SDS Stress Interpretation
+
+The SDS harness reports measured events. A cache hit is not enough. If `responseBytes` remains high, repeated clients are still consuming bandwidth unless the response artifact is served from edge/browser/WasmEdge host cache.
+
+For hot publish events, look at:
+
+- `hot-file-id-cache.responseBytes`
+- `raw-flatbuffer-retrieval.responseBytes`
+- `sql-projection-query.responseBytes`
+- `bulk-streaming-ingest.requestBytes`
+- aggregate `ingestRecordsPerSecond` and `minNodeIngestRecordsPerSecond`; every node must stay at or above 75 fps for production readiness
+- `bandwidth-constrained-streaming.requestBytes`
+- per-node `storageBytes` and storage fill percentage; every node must reach at least 95% of its configured storage target
+
+## Common Mistakes
+
+- Calling a run successful because query latency improved while response bytes remain unchanged.
+- Requiring fields like `file_identifier`, `indexed`, `key`, or `id` in SDS schemas.
+- Returning JSON for large repeated query results when raw FlatBuffer bytes are acceptable.
+- Building a process-local cache and assuming it solves 2M-user egress.
+- Omitting schema version, projection, format, or publish-event key from a cache key.

package/docs/response-artifacts.md

@@ -0,0 +1,86 @@
+# Response Artifacts
+
+FlatSQL reduces response bandwidth by producing cacheable response artifacts. The library owns deterministic byte output, cache keys, hashes, ETags, and chunk metadata. Deployment infrastructure owns CDN placement, global invalidation, routing, authorization, and quotas.
+
+## Boundary
+
+FlatSQL should do:
+
+- build deterministic cache keys in the C++/WASM core from query shape, schema version, format, projection, params, and publish-event key
+- encode raw FlatBuffer query results as size-prefixed byte streams
+- encode JSON results only when callers explicitly request JSON
+- produce content hashes and ETags for immutable artifacts
+- split artifacts into range-addressable chunks
+- expose small process-local caches for repeated query artifacts
+
+FlatSQL should not do:
+
+- add database-specific annotations to canonical SDS schemas
+- implement a CDN or global cache-invalidation service
+- decide authorization or tenant-specific cache policy
+- route users to regions or nodes
+- hide response byte cost behind a query-cache hit counter
+
+## Recommended FILE_ID Flow
+
+For Space Data Network publish events, `FILE_ID` is a network policy key, not a schema requirement.
+
+1. Ingest records for a publish event.
+2. Apply `FILE_ID` indexes through runtime policy or SDN index profiles.
+3. Query the publish event with a stable projection and format.
+4. Create a response artifact.
+5. Serve the artifact by `ETag` or content hash from edge, browser, WasmEdge host storage, or a CDN.
+6. Recompute only when the publish event, schema version, projection, or format changes.
+
+```ts
+import {
+  createQueryResponseArtifact,
+  MemoryResponseArtifactCache,
+} from 'flatsql/response';
+import initFlatSQL from 'flatsql/wasm';
+
+const flatsql = await initFlatSQL();
+const sql = "SELECT _data FROM PublishEventRecord WHERE FILE_ID = ?";
+const cacheKey = flatsql.buildResponseArtifactCacheKey('PublishEventRecord', '1', sql, {
+  format: 'raw-flatbuffer-stream',
+  publishEventKey: 'publish-1',
+  projection: ['_data'],
+  params: ['publish-1'],
+});
+const result = db.query(sql);
+const cache = new MemoryResponseArtifactCache();
+
+const artifact = cache.getOrCreateByKey(cacheKey, () => createQueryResponseArtifact(result, {
+  schemaName: 'PublishEventRecord',
+  schemaVersion: 1,
+  sql,
+  params: ['publish-1'],
+  format: 'raw-flatbuffer-stream',
+  publishEventKey: 'publish-1',
+  projection: ['_data'],
+  cacheKey,
+  chunkBytes: 1024 * 1024,
+}));
+
+console.log(cacheKey);
+console.log(artifact.metadata.etag);
+console.log(artifact.metadata.chunks);
+```
+
+## Production Checks
+
+Before relying on response artifacts in production:
+
+- verify `cacheKey` includes every dimension that can change bytes
+- verify browser and WasmEdge runtimes produce the same native response cache key
+- prefer `raw-flatbuffer-stream` for high-volume SDS responses
+- verify `responseBytes` and `flatbufferBytes`, not only query latency
+- tune native `configureQueryCache({ maxEntries, maxRows })` limits for hot FILE_ID query templates
+- set chunk sizes based on edge/cache/range-request behavior
+- cap in-process caches with `maxEntries` and `maxBytes`
+- keep authorization outside shared immutable artifacts unless all callers are allowed to read identical bytes
+- test hot `FILE_ID` traffic separately from cold fanout
+
+## Runtime Notes
+
+Browser deployments should call the native `buildResponseArtifactCacheKey(...)` binding through `flatsql/wasm` or `flatsql/standalone`, then map artifact metadata to HTTP cache headers and the browser Cache API where appropriate. WasmEdge deployments should call the same binding through `flatsql/artifacts/standalone` and map artifacts to host filesystem or edge-local object cache. FlatSQL response artifacts are intentionally plain bytes plus metadata so either runtime can use the same cache key and ETag semantics.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "flatsql",
-  "version": "0.4.3",
+  "version": "1.0.0",
   "description": "SQL queries over raw FlatBuffer storage - zero conversion, real-time indexing, pure streaming",
   "type": "module",
   "main": "dist/index.js",
@@ -18,6 +18,22 @@
       "types": "./wasm/wasi.d.ts",
       "import": "./wasm/wasi.js"
     },
+    "./standalone": {
+      "types": "./wasm/standalone.d.ts",
+      "import": "./wasm/standalone.js"
+    },
+    "./artifacts/standalone": {
+      "types": "./dist/artifacts/standalone-builder.d.ts",
+      "import": "./dist/artifacts/standalone-builder.js"
+    },
+    "./response": {
+      "types": "./dist/response/index.d.ts",
+      "import": "./dist/response/index.js"
+    },
+    "./standalone/wasmedge": {
+      "types": "./dist/standalone/index.d.ts",
+      "import": "./dist/standalone/index.js"
+    },
     "./wasi.wasm": "./wasm/flatsql-wasi.wasm",
     "./sdm": {
       "types": "./sdm/index.d.ts",
@@ -29,30 +45,43 @@
     "dist",
     "wasm/*.js",
     "wasm/*.wasm",
+    "wasm/*.json",
     "wasm/*.d.ts",
+    "wasm/native/*.cpp",
     "sdm/*.js",
     "sdm/*.wasm",
     "sdm/*.d.ts",
     "sdm/*.json",
+    "scripts/stress-docker.sh",
+    "stress/docker/*",
+    "stress/sds/*.mjs",
+    "docs/response-artifacts.md",
+    "docs/agents/*.md",
     "README.md",
     "LICENSE"
   ],
   "scripts": {
     "build": "tsc",
     "build:wasm": "bash scripts/build-wasm.sh",
-    "test": "node --experimental-vm-modules node_modules/jest/bin/jest.js",
+    "test": "node --experimental-wasm-exnref --experimental-vm-modules node_modules/jest/bin/jest.js",
     "test:wasm": "node wasm/test-bulk-ingest.mjs",
     "dev": "tsc --watch",
     "serve": "vite docs --port 8081",
     "bench": "node bench/flatsql-perf.mjs",
     "bench:perf": "node bench/flatsql-perf.mjs",
     "bench:perf:profile": "node bench/flatsql-perf.mjs --profile",
-    "prepublishOnly": "npm run build",
+    "prepublishOnly": "npm run build && npm run test:wasi && npm run test:wasmedge && npm test -- --runInBand --runTestsByPath test/package-exports.test.ts",
     "emsdk:install": "bash scripts/install-emsdk.sh",
     "emsdk:env": "echo 'Run: source packages/emsdk/emsdk_env.sh'",
-    "test:wasi": "node --experimental-vm-modules node_modules/jest/bin/jest.js --runTestsByPath test/wasi.test.ts",
+    "test:wasi": "node --experimental-wasm-exnref --experimental-vm-modules node_modules/jest/bin/jest.js --runTestsByPath test/wasi.test.ts test/standalone-wasm-imports.test.ts",
+    "test:wasmedge": "node --experimental-wasm-exnref --experimental-vm-modules node_modules/jest/bin/jest.js --runTestsByPath test/wasmedge-standalone.test.ts test/wasmedge-runner.test.ts",
     "test:cluster": "bash scripts/cluster-validation.sh",
-    "test:cluster:smoke": "bash scripts/cluster-validation.sh --duration=5 --stretch-duration=5 --readers=4 --stretch-readers=6"
+    "test:cluster:smoke": "bash scripts/cluster-validation.sh --duration=5 --stretch-duration=5 --readers=4 --stretch-readers=6",
+    "stress:sds:smoke": "bash scripts/stress-docker.sh smoke",
+    "stress:sds:docker-smoke": "bash scripts/stress-docker.sh docker-smoke",
+    "stress:sds:docker-fleet": "bash scripts/stress-docker.sh docker-fleet",
+    "stress:sds:full-config": "bash scripts/stress-docker.sh full-config",
+    "stress:sds:full": "bash scripts/stress-docker.sh full"
   },
   "repository": {
     "type": "git",
@@ -84,6 +113,7 @@
   "author": "Digital Arsenal <tj@digitalarsenal.io>",
   "license": "Apache-2.0",
   "dependencies": {
+    "flatbuffers": "^25.9.23",
     "flatc-wasm": "^26.1.32",
     "sql.js": "^1.10.0"
   },
@@ -96,7 +126,7 @@
     "vite": "^5.0.0"
   },
   "engines": {
-    "node": ">=18.0.0"
+    "node": ">=24.0.0"
   },
   "publishConfig": {
     "access": "public"

package/scripts/stress-docker.sh

@@ -0,0 +1,279 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(dirname "$SCRIPT_DIR")"
+
+COMMAND="smoke"
+if [[ $# -gt 0 ]]; then
+  COMMAND="$1"
+  shift
+fi
+
+RUN_ID="${RUN_ID:-sds-$(date -u +%Y%m%dT%H%M%SZ)}"
+SDS_SCHEMA_ROOT="${SDS_SCHEMA_ROOT:-$PROJECT_ROOT/../spacedatastandards.org/schema}"
+if [[ -n "${OUTPUT_DIR:-}" ]]; then
+  OUTPUT_DIR_EXPLICIT=1
+else
+  OUTPUT_DIR_EXPLICIT=0
+  OUTPUT_DIR="$PROJECT_ROOT/stress/results/$RUN_ID"
+fi
+
+EXTRA_RUNNER_ARGS=()
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --schema-root)
+      SDS_SCHEMA_ROOT="$2"
+      shift 2
+      ;;
+    --schema-root=*)
+      SDS_SCHEMA_ROOT="${1#*=}"
+      shift
+      ;;
+    --output-dir)
+      OUTPUT_DIR="$2"
+      OUTPUT_DIR_EXPLICIT=1
+      shift 2
+      ;;
+    --output-dir=*)
+      OUTPUT_DIR="${1#*=}"
+      OUTPUT_DIR_EXPLICIT=1
+      shift
+      ;;
+    --nodes)
+      NODE_COUNT="$2"
+      shift 2
+      ;;
+    --nodes=*)
+      NODE_COUNT="${1#*=}"
+      shift
+      ;;
+    --node-id-offset)
+      NODE_ID_OFFSET="$2"
+      shift 2
+      ;;
+    --node-id-offset=*)
+      NODE_ID_OFFSET="${1#*=}"
+      shift
+      ;;
+    --storage-gb)
+      NODE_STORAGE_GB="$2"
+      shift 2
+      ;;
+    --storage-gb=*)
+      NODE_STORAGE_GB="${1#*=}"
+      shift
+      ;;
+    --records-per-node)
+      RECORDS_PER_NODE="$2"
+      shift 2
+      ;;
+    --records-per-node=*)
+      RECORDS_PER_NODE="${1#*=}"
+      shift
+      ;;
+    --batch-bytes)
+      BATCH_BYTES="$2"
+      shift 2
+      ;;
+    --batch-bytes=*)
+      BATCH_BYTES="${1#*=}"
+      shift
+      ;;
+    --query-concurrency)
+      QUERY_CONCURRENCY="$2"
+      shift 2
+      ;;
+    --query-concurrency=*)
+      QUERY_CONCURRENCY="${1#*=}"
+      shift
+      ;;
+    --hot-query-ratio)
+      HOT_QUERY_RATIO="$2"
+      shift 2
+      ;;
+    --hot-query-ratio=*)
+      HOT_QUERY_RATIO="${1#*=}"
+      shift
+      ;;
+    --runtime)
+      RUNTIME="$2"
+      shift 2
+      ;;
+    --runtime=*)
+      RUNTIME="${1#*=}"
+      shift
+      ;;
+    --node-isolation)
+      NODE_ISOLATION="$2"
+      shift 2
+      ;;
+    --node-isolation=*)
+      NODE_ISOLATION="${1#*=}"
+      shift
+      ;;
+    --run-id)
+      RUN_ID="$2"
+      shift 2
+      ;;
+    --run-id=*)
+      RUN_ID="${1#*=}"
+      shift
+      ;;
+    *)
+      EXTRA_RUNNER_ARGS+=("$1")
+      shift
+      ;;
+  esac
+done
+if [[ "$OUTPUT_DIR_EXPLICIT" -eq 0 ]]; then
+  OUTPUT_DIR="$PROJECT_ROOT/stress/results/$RUN_ID"
+fi
+if [[ "${#EXTRA_RUNNER_ARGS[@]}" -gt 0 ]]; then
+  set -- "${EXTRA_RUNNER_ARGS[@]}"
+else
+  set --
+fi
+
+run_local_smoke() {
+  local runner_args=(
+    --mode "${STRESS_MODE:-smoke}"
+    --schema-root "$SDS_SCHEMA_ROOT"
+    --output-dir "$OUTPUT_DIR"
+    --nodes "${NODE_COUNT:-2}"
+    --storage-gb "${NODE_STORAGE_GB:-0.01}"
+    --records-per-node "${RECORDS_PER_NODE:-25}"
+    --run-id "$RUN_ID"
+  )
+  if [[ -n "${NODE_ID_OFFSET:-}" ]]; then runner_args+=(--node-id-offset "$NODE_ID_OFFSET"); fi
+  if [[ -n "${BATCH_BYTES:-}" ]]; then runner_args+=(--batch-bytes "$BATCH_BYTES"); fi
+  if [[ -n "${QUERY_CONCURRENCY:-}" ]]; then runner_args+=(--query-concurrency "$QUERY_CONCURRENCY"); fi
+  if [[ -n "${HOT_QUERY_RATIO:-}" ]]; then runner_args+=(--hot-query-ratio "$HOT_QUERY_RATIO"); fi
+  if [[ -n "${RUNTIME:-}" ]]; then runner_args+=(--runtime "$RUNTIME"); fi
+  if [[ -n "${NODE_ISOLATION:-}" ]]; then runner_args+=(--node-isolation "$NODE_ISOLATION"); fi
+  if [[ "$#" -gt 0 ]]; then runner_args+=("$@"); fi
+
+  npm run build
+  node "$PROJECT_ROOT/stress/sds/run.mjs" "${runner_args[@]}"
+}
+
+run_docker() {
+  docker info >/dev/null
+  local schema_root_abs
+  schema_root_abs="$(cd "$SDS_SCHEMA_ROOT" && pwd)"
+  mkdir -p "$OUTPUT_DIR"
+  local output_dir_abs
+  output_dir_abs="$(cd "$OUTPUT_DIR" && pwd)"
+  SDS_SCHEMA_ROOT_HOST="$schema_root_abs" \
+  OUTPUT_DIR_HOST="$output_dir_abs" \
+  RUN_ID="$RUN_ID" \
+  STRESS_MODE="${STRESS_MODE:-smoke}" \
+  NODE_COUNT="${NODE_COUNT:-2}" \
+  NODE_STORAGE_GB="${NODE_STORAGE_GB:-0.01}" \
+  RECORDS_PER_NODE="${RECORDS_PER_NODE:-25}" \
+  BATCH_BYTES="${BATCH_BYTES:-1048576}" \
+  QUERY_CONCURRENCY="${QUERY_CONCURRENCY:-4}" \
+  HOT_QUERY_RATIO="${HOT_QUERY_RATIO:-0.9}" \
+  RUNTIME="${RUNTIME:-standalone}" \
+  NODE_ISOLATION="${NODE_ISOLATION:-logical}" \
+  docker compose -f "$PROJECT_ROOT/stress/docker/docker-compose.yml" up \
+    --build \
+    --abort-on-container-exit \
+    --exit-code-from controller \
+    controller
+}
+
+prepare_container_fleet_compose() {
+  npm run build
+
+  local schema_root_abs
+  schema_root_abs="$(cd "$SDS_SCHEMA_ROOT" && pwd)"
+  mkdir -p "$OUTPUT_DIR"
+  local output_dir_abs
+  output_dir_abs="$(cd "$OUTPUT_DIR" && pwd)"
+  FLEET_COMPOSE_FILE="$output_dir_abs/docker-compose.full.yml"
+
+  node "$PROJECT_ROOT/stress/sds/write-compose.mjs" \
+    --output-file "$FLEET_COMPOSE_FILE" \
+    --project-root "$PROJECT_ROOT" \
+    --schema-root-host "$schema_root_abs" \
+    --output-dir-host "$output_dir_abs" \
+    --run-id "$RUN_ID" \
+    --mode "${STRESS_MODE:-smoke}" \
+    --nodes "${NODE_COUNT:-2}" \
+    --storage-gb "${NODE_STORAGE_GB:-0.01}" \
+    --records-per-node "${RECORDS_PER_NODE:-25}" \
+    --batch-bytes "${BATCH_BYTES:-1048576}" \
+    --query-concurrency "${QUERY_CONCURRENCY:-4}" \
+    --hot-query-ratio "${HOT_QUERY_RATIO:-0.9}" \
+    --runtime "${RUNTIME:-standalone}"
+}
+
+run_container_fleet() {
+  prepare_container_fleet_compose
+  docker info >/dev/null
+
+  local node_count="${NODE_COUNT:-2}"
+  local worker_services=()
+  local node_id
+  for ((node_id = 0; node_id < node_count; node_id++)); do
+    worker_services+=("worker-$node_id")
+  done
+
+  docker compose -f "$FLEET_COMPOSE_FILE" up --build "${worker_services[@]}"
+  docker compose -f "$FLEET_COMPOSE_FILE" build aggregator
+  docker compose -f "$FLEET_COMPOSE_FILE" run --no-deps --rm aggregator
+}
+
+run_full() {
+  if [[ "${CONFIRM_FULL_STRESS:-}" != "1" ]]; then
+    echo "Full stress requires CONFIRM_FULL_STRESS=1 because it targets 100 nodes and 1 GB/node." >&2
+    exit 2
+  fi
+
+  local parent
+  parent="$(dirname "$OUTPUT_DIR")"
+  mkdir -p "$parent"
+  local available_kb
+  available_kb="$(df -Pk "$parent" | awk 'NR == 2 { print $4 }')"
+  local required_kb=$((110 * 1024 * 1024))
+  if [[ "$available_kb" -lt "$required_kb" ]]; then
+    echo "Full stress requires at least 110 GB free near $parent; available KB: $available_kb." >&2
+    exit 3
+  fi
+
+  RECORDS_PER_NODE="${RECORDS_PER_NODE:-20000}"
+  QUERY_CONCURRENCY="${QUERY_CONCURRENCY:-64}"
+  HOT_QUERY_RATIO="${HOT_QUERY_RATIO:-0.01}"
+  STRESS_MODE=full NODE_COUNT=100 NODE_STORAGE_GB=1 NODE_ISOLATION=container run_container_fleet
+}
+
+case "$COMMAND" in
+  smoke)
+    run_local_smoke "$@"
+    ;;
+  docker-smoke)
+    run_docker "$@"
+    ;;
+  docker-fleet)
+    STRESS_MODE="${STRESS_MODE:-smoke}" NODE_ISOLATION=container run_container_fleet "$@"
+    ;;
+  full-config)
+    RECORDS_PER_NODE="${RECORDS_PER_NODE:-20000}" \
+      QUERY_CONCURRENCY="${QUERY_CONCURRENCY:-64}" \
+      HOT_QUERY_RATIO="${HOT_QUERY_RATIO:-0.01}" \
+      STRESS_MODE=full \
+      NODE_COUNT="${NODE_COUNT:-100}" \
+      NODE_STORAGE_GB="${NODE_STORAGE_GB:-1}" \
+      NODE_ISOLATION=container \
+      prepare_container_fleet_compose
+    ;;
+  full)
+    run_full "$@"
+    ;;
+  *)
+    echo "Usage: scripts/stress-docker.sh [smoke|docker-smoke|docker-fleet|full-config|full] [runner args]" >&2
+    exit 2
+    ;;
+esac