flatsql 0.4.2 → 1.0.0

package/stress/docker/Dockerfile

@@ -0,0 +1,11 @@
+FROM node:24-bookworm-slim
+
+WORKDIR /workspace
+
+COPY package.json package-lock.json ./
+RUN npm ci
+
+COPY . .
+RUN npm run build
+
+CMD ["node", "stress/sds/run.mjs", "--help"]

package/stress/docker/docker-compose.yml

@@ -0,0 +1,51 @@
+services:
+  controller:
+    build:
+      context: ../..
+      dockerfile: stress/docker/Dockerfile
+    environment:
+      STRESS_MODE: ${STRESS_MODE:-smoke}
+      RUN_ID: ${RUN_ID:-docker-smoke}
+      NODE_COUNT: ${NODE_COUNT:-2}
+      NODE_STORAGE_GB: ${NODE_STORAGE_GB:-0.01}
+      RECORDS_PER_NODE: ${RECORDS_PER_NODE:-25}
+      BATCH_BYTES: ${BATCH_BYTES:-1048576}
+      QUERY_CONCURRENCY: ${QUERY_CONCURRENCY:-4}
+      HOT_QUERY_RATIO: ${HOT_QUERY_RATIO:-0.9}
+      SDS_SCHEMA_ROOT: /sds/schema
+      OUTPUT_DIR: /stress-results
+      RUNTIME: ${RUNTIME:-standalone}
+      NODE_ISOLATION: ${NODE_ISOLATION:-logical}
+      STRESS_TRANSPORT: docker-compose
+    volumes:
+      - ${SDS_SCHEMA_ROOT_HOST:-../../spacedatastandards.org/schema}:/sds/schema:ro
+      - ${OUTPUT_DIR_HOST:-../results/docker-smoke}:/stress-results
+    command:
+      - node
+      - stress/sds/run.mjs
+      - --mode
+      - ${STRESS_MODE:-smoke}
+      - --schema-root
+      - /sds/schema
+      - --output-dir
+      - /stress-results
+      - --nodes
+      - ${NODE_COUNT:-2}
+      - --storage-gb
+      - ${NODE_STORAGE_GB:-0.01}
+      - --records-per-node
+      - ${RECORDS_PER_NODE:-25}
+      - --batch-bytes
+      - ${BATCH_BYTES:-1048576}
+      - --query-concurrency
+      - ${QUERY_CONCURRENCY:-4}
+      - --hot-query-ratio
+      - ${HOT_QUERY_RATIO:-0.9}
+      - --run-id
+      - ${RUN_ID:-docker-smoke}
+      - --runtime
+      - ${RUNTIME:-standalone}
+      - --node-isolation
+      - ${NODE_ISOLATION:-logical}
+      - --transport
+      - docker-compose

package/stress/sds/aggregate.mjs

@@ -0,0 +1,97 @@
+#!/usr/bin/env node
+
+import { readdir } from 'node:fs/promises';
+import { join, resolve } from 'node:path';
+import { aggregateStressRunArtifacts, assertProductionReadiness } from '../../dist/stress/index.js';
+
+function readOption(argv, name, defaultValue = undefined) {
+  const prefix = `--${name}=`;
+  let value = defaultValue;
+  for (let index = 0; index < argv.length; index++) {
+    const arg = argv[index];
+    if (arg === `--${name}`) {
+      value = argv[index + 1];
+      index++;
+    } else if (arg.startsWith(prefix)) {
+      value = arg.slice(prefix.length);
+    }
+  }
+  return value;
+}
+
+function readNumber(argv, name, defaultValue = undefined) {
+  const value = readOption(argv, name, defaultValue === undefined ? undefined : String(defaultValue));
+  if (value === undefined) {
+    return undefined;
+  }
+  const parsed = Number(value);
+  if (!Number.isFinite(parsed)) {
+    throw new Error(`Invalid numeric value for --${name}: ${value}`);
+  }
+  return parsed;
+}
+
+function printHelp() {
+  console.log(`Usage: node stress/sds/aggregate.mjs [options]
+
+Options:
+  --input-dir <path>         Directory containing node result subdirectories
+  --output-dir <path>        Directory for aggregate result artifacts
+  --run-id <id>
+  --mode smoke|full
+  --node-storage-gb <gb>
+  --records-per-node <count>
+  --transport in-process|docker-compose
+  --node-isolation logical|container
+`);
+}
+
+async function listInputDirs(inputRoot) {
+  const entries = await readdir(inputRoot, { withFileTypes: true });
+  return entries
+    .filter((entry) => entry.isDirectory())
+    .map((entry) => join(inputRoot, entry.name))
+    .sort();
+}
+
+async function main() {
+  const argv = process.argv.slice(2);
+  if (argv.includes('--help') || argv.includes('-h')) {
+    printHelp();
+    return;
+  }
+
+  const inputRoot = resolve(readOption(argv, 'input-dir', 'stress/results/nodes'));
+  const outputDir = resolve(readOption(argv, 'output-dir', 'stress/results/aggregate'));
+  const runId = readOption(argv, 'run-id', process.env.RUN_ID ?? `sds-aggregate-${Date.now()}`);
+  const mode = readOption(argv, 'mode', process.env.STRESS_MODE ?? 'smoke');
+  const inputDirs = await listInputDirs(inputRoot);
+
+  const result = await aggregateStressRunArtifacts({
+    inputDirs,
+    outputDir,
+    runId,
+    mode,
+    nodeStorageGb: readNumber(argv, 'node-storage-gb', process.env.NODE_STORAGE_GB) ?? 1,
+    recordsPerNode: readNumber(argv, 'records-per-node', process.env.RECORDS_PER_NODE) ?? 0,
+    transport: readOption(argv, 'transport', process.env.STRESS_TRANSPORT ?? 'docker-compose'),
+    nodeIsolation: readOption(argv, 'node-isolation', process.env.NODE_ISOLATION ?? 'container'),
+  });
+
+  console.log(`FlatSQL SDS aggregate ${result.manifest.mode} run: ${result.manifest.runId}`);
+  console.log(`Nodes: ${result.summary.nodes.length}`);
+  console.log(`Result directory: ${outputDir}`);
+  console.log(`Events: ${result.summary.eventCount}`);
+  console.log(`Request bytes: ${result.summary.totalRequestBytes}`);
+  console.log(`Response bytes: ${result.summary.totalResponseBytes}`);
+  console.log(`Cache hits/misses: ${result.summary.totalCacheHits}/${result.summary.totalCacheMisses}`);
+
+  if (result.manifest.mode === 'full') {
+    assertProductionReadiness(result.report);
+  }
+}
+
+main().catch((error) => {
+  console.error(error instanceof Error ? error.message : String(error));
+  process.exitCode = 1;
+});

package/stress/sds/run.mjs

@@ -0,0 +1,105 @@
+#!/usr/bin/env node
+
+import { resolve } from 'node:path';
+import { assertProductionReadiness, runSmokeHarness } from '../../dist/stress/index.js';
+
+function readOption(argv, name, defaultValue = undefined) {
+  const prefix = `--${name}=`;
+  let value = defaultValue;
+  for (let index = 0; index < argv.length; index++) {
+    const arg = argv[index];
+    if (arg === `--${name}`) {
+      value = argv[index + 1];
+      index++;
+    } else if (arg.startsWith(prefix)) {
+      value = arg.slice(prefix.length);
+    }
+  }
+  return value;
+}
+
+function readNumber(argv, name, defaultValue = undefined) {
+  const value = readOption(argv, name, defaultValue === undefined ? undefined : String(defaultValue));
+  if (value === undefined) {
+    return undefined;
+  }
+  const parsed = Number(value);
+  if (!Number.isFinite(parsed)) {
+    throw new Error(`Invalid numeric value for --${name}: ${value}`);
+  }
+  return parsed;
+}
+
+function printHelp() {
+  console.log(`Usage: node stress/sds/run.mjs [options]
+
+Options:
+  --mode smoke|full
+  --schema-root <path>
+  --output-dir <path>
+  --nodes <count>
+  --node-id-offset <count>
+  --storage-gb <gb>
+  --records-per-node <count>
+  --batch-bytes <bytes>
+  --query-concurrency <count>
+  --hot-query-ratio <ratio>
+  --runtime standalone|wasmedge|native
+  --transport in-process|docker-compose
+  --node-isolation logical|container
+  --skip-production-gate
+  --run-id <id>
+`);
+}
+
+async function main() {
+  const argv = process.argv.slice(2);
+  if (argv.includes('--help') || argv.includes('-h')) {
+    printHelp();
+    return;
+  }
+
+  const mode = readOption(argv, 'mode', process.env.STRESS_MODE ?? 'smoke');
+  const runId = readOption(argv, 'run-id', process.env.RUN_ID ?? `sds-${mode}-${Date.now()}`);
+  const schemaRoot = resolve(
+    readOption(argv, 'schema-root', process.env.SDS_SCHEMA_ROOT ?? '../spacedatastandards.org/schema')
+  );
+  const outputDir = resolve(
+    readOption(argv, 'output-dir', process.env.OUTPUT_DIR ?? `stress/results/${runId}`)
+  );
+
+  const result = await runSmokeHarness({
+    mode,
+    runId,
+    schemaRoot,
+    outputDir,
+    nodeCount: readNumber(argv, 'nodes', process.env.NODE_COUNT),
+    nodeIdOffset: readNumber(argv, 'node-id-offset', process.env.NODE_ID_OFFSET),
+    nodeStorageGb: readNumber(argv, 'storage-gb', process.env.NODE_STORAGE_GB),
+    recordsPerNode: readNumber(argv, 'records-per-node', process.env.RECORDS_PER_NODE),
+    batchBytes: readNumber(argv, 'batch-bytes', process.env.BATCH_BYTES),
+    queryConcurrency: readNumber(argv, 'query-concurrency', process.env.QUERY_CONCURRENCY),
+    hotQueryRatio: readNumber(argv, 'hot-query-ratio', process.env.HOT_QUERY_RATIO),
+    runtime: readOption(argv, 'runtime', process.env.RUNTIME),
+    transport: readOption(argv, 'transport', process.env.STRESS_TRANSPORT),
+    nodeIsolation: readOption(argv, 'node-isolation', process.env.NODE_ISOLATION),
+  });
+
+  console.log(`FlatSQL SDS stress ${result.manifest.mode} run: ${result.manifest.runId}`);
+  console.log(`Schemas: ${result.manifest.schemaCount}`);
+  console.log(`Nodes: ${result.manifest.nodeCount}`);
+  console.log(`Result directory: ${outputDir}`);
+  console.log(`Events: ${result.summary.eventCount}`);
+  console.log(`Request bytes: ${result.summary.totalRequestBytes}`);
+  console.log(`Response bytes: ${result.summary.totalResponseBytes}`);
+  console.log(`Cache hits/misses: ${result.summary.totalCacheHits}/${result.summary.totalCacheMisses}`);
+
+  if (result.manifest.mode === 'full' && !argv.includes('--skip-production-gate')) {
+    assertProductionReadiness(result.report);
+  }
+}
+
+main().catch((error) => {
+  console.error(error instanceof Error ? error.message : String(error));
+  process.exitCode = 1;
+});

package/stress/sds/write-compose.mjs

@@ -0,0 +1,96 @@
+#!/usr/bin/env node
+
+import { mkdir, writeFile } from 'node:fs/promises';
+import { dirname, resolve } from 'node:path';
+import { buildFullStressDockerCompose } from '../../dist/stress/index.js';
+
+function readOption(argv, name, defaultValue = undefined) {
+  const prefix = `--${name}=`;
+  let value = defaultValue;
+  for (let index = 0; index < argv.length; index++) {
+    const arg = argv[index];
+    if (arg === `--${name}`) {
+      value = argv[index + 1];
+      index++;
+    } else if (arg.startsWith(prefix)) {
+      value = arg.slice(prefix.length);
+    }
+  }
+  return value;
+}
+
+function readNumber(argv, name, defaultValue = undefined) {
+  const value = readOption(argv, name, defaultValue === undefined ? undefined : String(defaultValue));
+  if (value === undefined) {
+    return undefined;
+  }
+  const parsed = Number(value);
+  if (!Number.isFinite(parsed)) {
+    throw new Error(`Invalid numeric value for --${name}: ${value}`);
+  }
+  return parsed;
+}
+
+function printHelp() {
+  console.log(`Usage: node stress/sds/write-compose.mjs [options]
+
+Options:
+  --output-file <path>
+  --project-root <path>
+  --schema-root-host <path>
+  --output-dir-host <path>
+  --run-id <id>
+  --mode smoke|full
+  --nodes <count>
+  --storage-gb <gb>
+  --records-per-node <count>
+  --batch-bytes <bytes>
+  --query-concurrency <count>
+  --hot-query-ratio <ratio>
+  --runtime standalone|wasmedge|native
+`);
+}
+
+async function main() {
+  const argv = process.argv.slice(2);
+  if (argv.includes('--help') || argv.includes('-h')) {
+    printHelp();
+    return;
+  }
+
+  const outputFile = resolve(readOption(argv, 'output-file', 'stress/results/docker-compose.full.yml'));
+  const projectRoot = resolve(readOption(argv, 'project-root', process.cwd()));
+  const schemaRootHost = resolve(readOption(argv, 'schema-root-host', '../spacedatastandards.org/schema'));
+  const outputDirHost = resolve(readOption(argv, 'output-dir-host', 'stress/results/full'));
+  const nodeCount = readNumber(argv, 'nodes', process.env.NODE_COUNT) ?? 100;
+
+  const compose = buildFullStressDockerCompose({
+    projectRoot,
+    schemaRootHost,
+    outputDirHost,
+    runId: readOption(argv, 'run-id', process.env.RUN_ID ?? `sds-full-${Date.now()}`),
+    mode: readOption(argv, 'mode', process.env.STRESS_MODE ?? 'full'),
+    nodeCount,
+    nodeStorageGb: readNumber(argv, 'storage-gb', process.env.NODE_STORAGE_GB) ?? 1,
+    recordsPerNode: readNumber(argv, 'records-per-node', process.env.RECORDS_PER_NODE) ?? 2900000,
+    batchBytes: readNumber(argv, 'batch-bytes', process.env.BATCH_BYTES) ?? 1048576,
+    queryConcurrency: readNumber(argv, 'query-concurrency', process.env.QUERY_CONCURRENCY) ?? 64,
+    hotQueryRatio: readNumber(argv, 'hot-query-ratio', process.env.HOT_QUERY_RATIO) ?? 0.01,
+    runtime: readOption(argv, 'runtime', process.env.RUNTIME ?? 'standalone'),
+  });
+
+  await mkdir(dirname(outputFile), { recursive: true });
+  await mkdir(outputDirHost, { recursive: true });
+  await mkdir(`${outputDirHost}/nodes`, { recursive: true });
+  await mkdir(`${outputDirHost}/aggregate`, { recursive: true });
+  await Promise.all(
+    Array.from({ length: nodeCount }, (_, nodeId) => mkdir(`${outputDirHost}/nodes/node-${nodeId}`, { recursive: true }))
+  );
+  await writeFile(outputFile, compose);
+  console.log(outputFile);
+}
+
+main().catch((error) => {
+  console.error(error instanceof Error ? error.message : String(error));
+  process.exitCode = 1;
+});