flatsql 3.11.1 → 3.12.0

package/stress/docker/docker-compose.yml

@@ -1,51 +0,0 @@
-services:
-  controller:
-    build:
-      context: ../..
-      dockerfile: stress/docker/Dockerfile
-    environment:
-      STRESS_MODE: ${STRESS_MODE:-smoke}
-      RUN_ID: ${RUN_ID:-docker-smoke}
-      NODE_COUNT: ${NODE_COUNT:-2}
-      NODE_STORAGE_GB: ${NODE_STORAGE_GB:-0.01}
-      RECORDS_PER_NODE: ${RECORDS_PER_NODE:-25}
-      BATCH_BYTES: ${BATCH_BYTES:-1048576}
-      QUERY_CONCURRENCY: ${QUERY_CONCURRENCY:-4}
-      HOT_QUERY_RATIO: ${HOT_QUERY_RATIO:-0.9}
-      SDS_SCHEMA_ROOT: /sds/schema
-      OUTPUT_DIR: /stress-results
-      RUNTIME: ${RUNTIME:-standalone}
-      NODE_ISOLATION: ${NODE_ISOLATION:-logical}
-      STRESS_TRANSPORT: docker-compose
-    volumes:
-      - ${SDS_SCHEMA_ROOT_HOST:-../../spacedatastandards.org/schema}:/sds/schema:ro
-      - ${OUTPUT_DIR_HOST:-../results/docker-smoke}:/stress-results
-    command:
-      - node
-      - stress/sds/run.mjs
-      - --mode
-      - ${STRESS_MODE:-smoke}
-      - --schema-root
-      - /sds/schema
-      - --output-dir
-      - /stress-results
-      - --nodes
-      - ${NODE_COUNT:-2}
-      - --storage-gb
-      - ${NODE_STORAGE_GB:-0.01}
-      - --records-per-node
-      - ${RECORDS_PER_NODE:-25}
-      - --batch-bytes
-      - ${BATCH_BYTES:-1048576}
-      - --query-concurrency
-      - ${QUERY_CONCURRENCY:-4}
-      - --hot-query-ratio
-      - ${HOT_QUERY_RATIO:-0.9}
-      - --run-id
-      - ${RUN_ID:-docker-smoke}
-      - --runtime
-      - ${RUNTIME:-standalone}
-      - --node-isolation
-      - ${NODE_ISOLATION:-logical}
-      - --transport
-      - docker-compose

package/stress/sds/aggregate.mjs

@@ -1,97 +0,0 @@
-#!/usr/bin/env node
-
-import { readdir } from 'node:fs/promises';
-import { join, resolve } from 'node:path';
-import { aggregateStressRunArtifacts, assertProductionReadiness } from '../../dist/stress/index.js';
-
-function readOption(argv, name, defaultValue = undefined) {
-  const prefix = `--${name}=`;
-  let value = defaultValue;
-  for (let index = 0; index < argv.length; index++) {
-    const arg = argv[index];
-    if (arg === `--${name}`) {
-      value = argv[index + 1];
-      index++;
-    } else if (arg.startsWith(prefix)) {
-      value = arg.slice(prefix.length);
-    }
-  }
-  return value;
-}
-
-function readNumber(argv, name, defaultValue = undefined) {
-  const value = readOption(argv, name, defaultValue === undefined ? undefined : String(defaultValue));
-  if (value === undefined) {
-    return undefined;
-  }
-  const parsed = Number(value);
-  if (!Number.isFinite(parsed)) {
-    throw new Error(`Invalid numeric value for --${name}: ${value}`);
-  }
-  return parsed;
-}
-
-function printHelp() {
-  console.log(`Usage: node stress/sds/aggregate.mjs [options]
-
-Options:
-  --input-dir <path>         Directory containing node result subdirectories
-  --output-dir <path>        Directory for aggregate result artifacts
-  --run-id <id>
-  --mode smoke|full
-  --node-storage-gb <gb>
-  --records-per-node <count>
-  --transport in-process|docker-compose
-  --node-isolation logical|container
-`);
-}
-
-async function listInputDirs(inputRoot) {
-  const entries = await readdir(inputRoot, { withFileTypes: true });
-  return entries
-    .filter((entry) => entry.isDirectory())
-    .map((entry) => join(inputRoot, entry.name))
-    .sort();
-}
-
-async function main() {
-  const argv = process.argv.slice(2);
-  if (argv.includes('--help') || argv.includes('-h')) {
-    printHelp();
-    return;
-  }
-
-  const inputRoot = resolve(readOption(argv, 'input-dir', 'stress/results/nodes'));
-  const outputDir = resolve(readOption(argv, 'output-dir', 'stress/results/aggregate'));
-  const runId = readOption(argv, 'run-id', process.env.RUN_ID ?? `sds-aggregate-${Date.now()}`);
-  const mode = readOption(argv, 'mode', process.env.STRESS_MODE ?? 'smoke');
-  const inputDirs = await listInputDirs(inputRoot);
-
-  const result = await aggregateStressRunArtifacts({
-    inputDirs,
-    outputDir,
-    runId,
-    mode,
-    nodeStorageGb: readNumber(argv, 'node-storage-gb', process.env.NODE_STORAGE_GB) ?? 1,
-    recordsPerNode: readNumber(argv, 'records-per-node', process.env.RECORDS_PER_NODE) ?? 0,
-    transport: readOption(argv, 'transport', process.env.STRESS_TRANSPORT ?? 'docker-compose'),
-    nodeIsolation: readOption(argv, 'node-isolation', process.env.NODE_ISOLATION ?? 'container'),
-  });
-
-  console.log(`FlatSQL SDS aggregate ${result.manifest.mode} run: ${result.manifest.runId}`);
-  console.log(`Nodes: ${result.summary.nodes.length}`);
-  console.log(`Result directory: ${outputDir}`);
-  console.log(`Events: ${result.summary.eventCount}`);
-  console.log(`Request bytes: ${result.summary.totalRequestBytes}`);
-  console.log(`Response bytes: ${result.summary.totalResponseBytes}`);
-  console.log(`Cache hits/misses: ${result.summary.totalCacheHits}/${result.summary.totalCacheMisses}`);
-
-  if (result.manifest.mode === 'full') {
-    assertProductionReadiness(result.report);
-  }
-}
-
-main().catch((error) => {
-  console.error(error instanceof Error ? error.message : String(error));
-  process.exitCode = 1;
-});

package/stress/sds/run.mjs

@@ -1,105 +0,0 @@
-#!/usr/bin/env node
-
-import { resolve } from 'node:path';
-import { assertProductionReadiness, runSmokeHarness } from '../../dist/stress/index.js';
-
-function readOption(argv, name, defaultValue = undefined) {
-  const prefix = `--${name}=`;
-  let value = defaultValue;
-  for (let index = 0; index < argv.length; index++) {
-    const arg = argv[index];
-    if (arg === `--${name}`) {
-      value = argv[index + 1];
-      index++;
-    } else if (arg.startsWith(prefix)) {
-      value = arg.slice(prefix.length);
-    }
-  }
-  return value;
-}
-
-function readNumber(argv, name, defaultValue = undefined) {
-  const value = readOption(argv, name, defaultValue === undefined ? undefined : String(defaultValue));
-  if (value === undefined) {
-    return undefined;
-  }
-  const parsed = Number(value);
-  if (!Number.isFinite(parsed)) {
-    throw new Error(`Invalid numeric value for --${name}: ${value}`);
-  }
-  return parsed;
-}
-
-function printHelp() {
-  console.log(`Usage: node stress/sds/run.mjs [options]
-
-Options:
-  --mode smoke|full
-  --schema-root <path>
-  --output-dir <path>
-  --nodes <count>
-  --node-id-offset <count>
-  --storage-gb <gb>
-  --records-per-node <count>
-  --batch-bytes <bytes>
-  --query-concurrency <count>
-  --hot-query-ratio <ratio>
-  --runtime standalone|wasmedge|native
-  --transport in-process|docker-compose
-  --node-isolation logical|container
-  --skip-production-gate
-  --run-id <id>
-`);
-}
-
-async function main() {
-  const argv = process.argv.slice(2);
-  if (argv.includes('--help') || argv.includes('-h')) {
-    printHelp();
-    return;
-  }
-
-  const mode = readOption(argv, 'mode', process.env.STRESS_MODE ?? 'smoke');
-  const runId = readOption(argv, 'run-id', process.env.RUN_ID ?? `sds-${mode}-${Date.now()}`);
-  const schemaRoot = resolve(
-    readOption(argv, 'schema-root', process.env.SDS_SCHEMA_ROOT ?? '../spacedatastandards.org/schema')
-  );
-  const outputDir = resolve(
-    readOption(argv, 'output-dir', process.env.OUTPUT_DIR ?? `stress/results/${runId}`)
-  );
-
-  const result = await runSmokeHarness({
-    mode,
-    runId,
-    schemaRoot,
-    outputDir,
-    nodeCount: readNumber(argv, 'nodes', process.env.NODE_COUNT),
-    nodeIdOffset: readNumber(argv, 'node-id-offset', process.env.NODE_ID_OFFSET),
-    nodeStorageGb: readNumber(argv, 'storage-gb', process.env.NODE_STORAGE_GB),
-    recordsPerNode: readNumber(argv, 'records-per-node', process.env.RECORDS_PER_NODE),
-    batchBytes: readNumber(argv, 'batch-bytes', process.env.BATCH_BYTES),
-    queryConcurrency: readNumber(argv, 'query-concurrency', process.env.QUERY_CONCURRENCY),
-    hotQueryRatio: readNumber(argv, 'hot-query-ratio', process.env.HOT_QUERY_RATIO),
-    runtime: readOption(argv, 'runtime', process.env.RUNTIME),
-    transport: readOption(argv, 'transport', process.env.STRESS_TRANSPORT),
-    nodeIsolation: readOption(argv, 'node-isolation', process.env.NODE_ISOLATION),
-  });
-
-  console.log(`FlatSQL SDS stress ${result.manifest.mode} run: ${result.manifest.runId}`);
-  console.log(`Schemas: ${result.manifest.schemaCount}`);
-  console.log(`Nodes: ${result.manifest.nodeCount}`);
-  console.log(`Result directory: ${outputDir}`);
-  console.log(`Events: ${result.summary.eventCount}`);
-  console.log(`Request bytes: ${result.summary.totalRequestBytes}`);
-  console.log(`Response bytes: ${result.summary.totalResponseBytes}`);
-  console.log(`Cache hits/misses: ${result.summary.totalCacheHits}/${result.summary.totalCacheMisses}`);
-
-  if (result.manifest.mode === 'full' && !argv.includes('--skip-production-gate')) {
-    assertProductionReadiness(result.report);
-  }
-}
-
-main().catch((error) => {
-  console.error(error instanceof Error ? error.message : String(error));
-  process.exitCode = 1;
-});

package/stress/sds/write-compose.mjs

@@ -1,96 +0,0 @@
-#!/usr/bin/env node
-
-import { mkdir, writeFile } from 'node:fs/promises';
-import { dirname, resolve } from 'node:path';
-import { buildFullStressDockerCompose } from '../../dist/stress/index.js';
-
-function readOption(argv, name, defaultValue = undefined) {
-  const prefix = `--${name}=`;
-  let value = defaultValue;
-  for (let index = 0; index < argv.length; index++) {
-    const arg = argv[index];
-    if (arg === `--${name}`) {
-      value = argv[index + 1];
-      index++;
-    } else if (arg.startsWith(prefix)) {
-      value = arg.slice(prefix.length);
-    }
-  }
-  return value;
-}
-
-function readNumber(argv, name, defaultValue = undefined) {
-  const value = readOption(argv, name, defaultValue === undefined ? undefined : String(defaultValue));
-  if (value === undefined) {
-    return undefined;
-  }
-  const parsed = Number(value);
-  if (!Number.isFinite(parsed)) {
-    throw new Error(`Invalid numeric value for --${name}: ${value}`);
-  }
-  return parsed;
-}
-
-function printHelp() {
-  console.log(`Usage: node stress/sds/write-compose.mjs [options]
-
-Options:
-  --output-file <path>
-  --project-root <path>
-  --schema-root-host <path>
-  --output-dir-host <path>
-  --run-id <id>
-  --mode smoke|full
-  --nodes <count>
-  --storage-gb <gb>
-  --records-per-node <count>
-  --batch-bytes <bytes>
-  --query-concurrency <count>
-  --hot-query-ratio <ratio>
-  --runtime standalone|wasmedge|native
-`);
-}
-
-async function main() {
-  const argv = process.argv.slice(2);
-  if (argv.includes('--help') || argv.includes('-h')) {
-    printHelp();
-    return;
-  }
-
-  const outputFile = resolve(readOption(argv, 'output-file', 'stress/results/docker-compose.full.yml'));
-  const projectRoot = resolve(readOption(argv, 'project-root', process.cwd()));
-  const schemaRootHost = resolve(readOption(argv, 'schema-root-host', '../spacedatastandards.org/schema'));
-  const outputDirHost = resolve(readOption(argv, 'output-dir-host', 'stress/results/full'));
-  const nodeCount = readNumber(argv, 'nodes', process.env.NODE_COUNT) ?? 100;
-
-  const compose = buildFullStressDockerCompose({
-    projectRoot,
-    schemaRootHost,
-    outputDirHost,
-    runId: readOption(argv, 'run-id', process.env.RUN_ID ?? `sds-full-${Date.now()}`),
-    mode: readOption(argv, 'mode', process.env.STRESS_MODE ?? 'full'),
-    nodeCount,
-    nodeStorageGb: readNumber(argv, 'storage-gb', process.env.NODE_STORAGE_GB) ?? 1,
-    recordsPerNode: readNumber(argv, 'records-per-node', process.env.RECORDS_PER_NODE) ?? 2900000,
-    batchBytes: readNumber(argv, 'batch-bytes', process.env.BATCH_BYTES) ?? 1048576,
-    queryConcurrency: readNumber(argv, 'query-concurrency', process.env.QUERY_CONCURRENCY) ?? 64,
-    hotQueryRatio: readNumber(argv, 'hot-query-ratio', process.env.HOT_QUERY_RATIO) ?? 0.01,
-    runtime: readOption(argv, 'runtime', process.env.RUNTIME ?? 'standalone'),
-  });
-
-  await mkdir(dirname(outputFile), { recursive: true });
-  await mkdir(outputDirHost, { recursive: true });
-  await mkdir(`${outputDirHost}/nodes`, { recursive: true });
-  await mkdir(`${outputDirHost}/aggregate`, { recursive: true });
-  await Promise.all(
-    Array.from({ length: nodeCount }, (_, nodeId) => mkdir(`${outputDirHost}/nodes/node-${nodeId}`, { recursive: true }))
-  );
-  await writeFile(outputFile, compose);
-  console.log(outputFile);
-}
-
-main().catch((error) => {
-  console.error(error instanceof Error ? error.message : String(error));
-  process.exitCode = 1;
-});

package/wasm/flatsql-client.js

@@ -1,280 +0,0 @@
-/**
- * FlatSQL Worker Client
- * Provides a clean async API for communicating with the FlatSQL worker.
- * Works in: Browser, Node.js, Deno
- *
- * Usage:
- *   import { FlatSQLClient, FlatSQLWorkerDatabase } from './flatsql-client.js';
- *
- *   const client = new FlatSQLClient();
- *   await client.init();
- *
- *   const db = await client.createDatabase(schema, 'mydb');
- *   await db.registerFileId('USER', 'User');
- *   const result = await db.query('SELECT * FROM User');
- */
-
-// Environment detection
-const isBrowser = typeof window !== 'undefined' || (typeof self !== 'undefined' && typeof importScripts === 'undefined');
-const isNode = typeof process !== 'undefined' && process.versions && process.versions.node;
-const isDeno = typeof Deno !== 'undefined';
-
-/**
- * Create a worker based on the environment
- */
-async function createWorker(workerPath) {
-    if (isNode) {
-        const { Worker } = await import('worker_threads');
-        return new Worker(workerPath);
-    } else if (isDeno) {
-        return new Worker(new URL(workerPath, import.meta.url), { type: 'module' });
-    } else {
-        // Browser
-        return new Worker(workerPath, { type: 'module' });
-    }
-}
-
-/**
- * Setup message handlers for the worker
- */
-function setupWorkerHandlers(worker, handlers) {
-    if (isNode) {
-        worker.on('message', handlers.onMessage);
-        worker.on('error', handlers.onError);
-        worker.on('exit', handlers.onExit);
-    } else {
-        worker.onmessage = (e) => handlers.onMessage(e.data);
-        worker.onerror = handlers.onError;
-    }
-}
-
-/**
- * FlatSQL Worker Client
- * Manages communication with the FlatSQL worker.
- */
-export class FlatSQLClient {
-    constructor(workerPath = './flatsql.worker.js') {
-        this.workerPath = workerPath;
-        this.worker = null;
-        this.pending = new Map();
-        this.nextId = 1;
-        this.ready = false;
-        this.readyPromise = null;
-        this.databases = new Map();
-    }
-
-    /**
-     * Initialize the client and worker
-     */
-    async init() {
-        if (this.worker) return this;
-
-        this.worker = await createWorker(this.workerPath);
-
-        this.readyPromise = new Promise((resolve, reject) => {
-            const timeout = setTimeout(() => {
-                reject(new Error('Worker initialization timeout'));
-            }, 30000);
-
-            const handlers = {
-                onMessage: (data) => {
-                    if (data.type === 'ready') {
-                        clearTimeout(timeout);
-                        this.ready = true;
-                        resolve();
-                        return;
-                    }
-
-                    const { id, success, result, error } = data;
-                    const pending = this.pending.get(id);
-                    if (pending) {
-                        this.pending.delete(id);
-                        if (success) {
-                            pending.resolve(result);
-                        } else {
-                            pending.reject(new Error(error));
-                        }
-                    }
-                },
-                onError: (err) => {
-                    clearTimeout(timeout);
-                    reject(err);
-                },
-                onExit: (code) => {
-                    if (code !== 0) {
-                        reject(new Error(`Worker exited with code ${code}`));
-                    }
-                }
-            };
-
-            setupWorkerHandlers(this.worker, handlers);
-        });
-
-        await this.readyPromise;
-
-        // Initialize WASM in worker
-        await this.call('init');
-
-        return this;
-    }
-
-    /**
-     * Call a method on the worker
-     */
-    async call(method, params = {}) {
-        if (!this.worker) {
-            throw new Error('Client not initialized. Call init() first.');
-        }
-
-        const id = this.nextId++;
-
-        return new Promise((resolve, reject) => {
-            this.pending.set(id, { resolve, reject });
-            this.worker.postMessage({ id, method, params });
-        });
-    }
-
-    /**
-     * Create a new database
-     */
-    async createDatabase(schema, name = 'default') {
-        const dbId = `db_${this.nextId++}`;
-        await this.call('createDatabase', { id: dbId, schema, name });
-        const db = new FlatSQLWorkerDatabase(this, dbId);
-        this.databases.set(dbId, db);
-        return db;
-    }
-
-    /**
-     * Create a test User FlatBuffer (for demos)
-     */
-    async createTestUser(id, name, email, age) {
-        const result = await this.call('createTestUser', { id, name, email, age });
-        return new Uint8Array(result.data);
-    }
-
-    /**
-     * Create a test Post FlatBuffer (for demos)
-     */
-    async createTestPost(id, userId, title) {
-        const result = await this.call('createTestPost', { id, userId, title });
-        return new Uint8Array(result.data);
-    }
-
-    /**
-     * Terminate the worker
-     */
-    terminate() {
-        if (this.worker) {
-            if (isNode) {
-                this.worker.terminate();
-            } else {
-                this.worker.terminate();
-            }
-            this.worker = null;
-            this.ready = false;
-        }
-    }
-}
-
-/**
- * FlatSQL Database accessed through a worker
- */
-export class FlatSQLWorkerDatabase {
-    constructor(client, dbId) {
-        this._client = client;
-        this._dbId = dbId;
-    }
-
-    /**
-     * Register a file ID for routing FlatBuffers to tables
-     */
-    async registerFileId(fileId, tableName) {
-        await this._client.call('registerFileId', {
-            dbId: this._dbId,
-            fileId,
-            tableName
-        });
-    }
-
-    /**
-     * Enable built-in demo extractors (User/Post)
-     */
-    async enableDemoExtractors() {
-        await this._client.call('enableDemoExtractors', { dbId: this._dbId });
-    }
-
-    /**
-     * Execute a SQL query
-     */
-    async query(sql) {
-        return await this._client.call('query', { dbId: this._dbId, sql });
-    }
-
-    /**
-     * Ingest a size-prefixed FlatBuffer stream
-     */
-    async ingest(data) {
-        const result = await this._client.call('ingest', {
-            dbId: this._dbId,
-            data: Array.from(data)
-        });
-        return result.bytesConsumed;
-    }
-
-    /**
-     * Ingest a single FlatBuffer (without size prefix)
-     */
-    async ingestOne(data) {
-        const result = await this._client.call('ingestOne', {
-            dbId: this._dbId,
-            data: Array.from(data)
-        });
-        return result.rowId;
-    }
-
-    /**
-     * Ingest multiple FlatBuffers (builds stream internally)
-     */
-    async ingestStream(buffers) {
-        const result = await this._client.call('ingestStream', {
-            dbId: this._dbId,
-            buffers: buffers.map(b => Array.from(b))
-        });
-        return result;
-    }
-
-    /**
-     * Export database as binary blob
-     */
-    async exportData() {
-        const result = await this._client.call('exportData', { dbId: this._dbId });
-        return new Uint8Array(result.data);
-    }
-
-    /**
-     * List all tables
-     */
-    async listTables() {
-        const result = await this._client.call('listTables', { dbId: this._dbId });
-        return result.tables;
-    }
-
-    /**
-     * Get database statistics
-     */
-    async getStats() {
-        const result = await this._client.call('getStats', { dbId: this._dbId });
-        return result.stats;
-    }
-
-    /**
-     * Delete the database and free resources
-     */
-    async delete() {
-        await this._client.call('deleteDatabase', { dbId: this._dbId });
-        this._client.databases.delete(this._dbId);
-    }
-}
-
-export default FlatSQLClient;