@gns-foundation/hive-worker 0.1.10 → 0.5.0

package/src/mobydb.ts

@@ -0,0 +1,558 @@
+// ============================================================
+// Phase 4: MobyDB Embedded Storage
+// File: packages/hive-worker/src/mobydb.ts
+//
+// Local storage engine for the Hive worker.
+// Phase 4a: SQLite (better-sqlite3)
+// Phase 4b: RocksDB (Rust sidecar)
+//
+// Every compute job (inference, tile, etc.) writes a record
+// locally. Records are sealed into epochs with Merkle roots.
+// ============================================================
+
+import Database from 'better-sqlite3';
+import * as crypto from 'crypto';
+import * as path from 'path';
+import * as fs from 'fs';
+
+// ─── Config ─────────────────────────────────
+const HIVE_DIR = process.env.HIVE_DIR || path.join(process.env.HOME || '/tmp', '.hive');
+const DB_PATH = path.join(HIVE_DIR, 'mobydb', 'records.db');
+const EPOCH_DURATION_MS = 3600_000; // 1 hour
+const EPOCH_ZERO = new Date('2026-01-01T00:00:00Z').getTime();
+
+// ─── Types ──────────────────────────────────
+
+export type CollectionType =
+  | 'Breadcrumb'
+  | 'Telemetry'
+  | 'Event'
+  | 'Territory'
+  | 'Relationship'
+  | 'Aggregate'
+  | 'Imagery'
+  | 'Inference'
+  | 'Tile'
+  | 'ComputeJob';
+
+export interface MobyDBAddress {
+  h3_cell: string;
+  epoch: number;
+  public_key: string;
+}
+
+export interface MobyDBRecord {
+  id: string;
+  address: MobyDBAddress;
+  collection_type: CollectionType;
+  payload_type: string;
+  data: Record<string, any>;
+  signature?: string;
+  trust_tier: string;
+  written_at_ms: number;
+  record_hash: string;
+  sealed: boolean;
+  epoch_merkle_root?: string;
+}
+
+export interface EpochSeal {
+  epoch: number;
+  h3_cell: string;
+  worker_pk: string;
+  record_count: number;
+  merkle_root: string;
+  prev_epoch_hash: string;
+  sealed_at_ms: number;
+  signature?: string;
+}
+
+// ─── Hash Helpers ───────────────────────────
+
+function sha256(input: string): string {
+  return crypto.createHash('sha256').update(input, 'utf8').digest('hex');
+}
+
+function computeRecordHash(record: Omit<MobyDBRecord, 'record_hash' | 'sealed' | 'epoch_merkle_root'>): string {
+  const canonical = JSON.stringify({
+    address: record.address,
+    collection_type: record.collection_type,
+    payload_type: record.payload_type,
+    data: record.data,
+    written_at_ms: record.written_at_ms,
+  });
+  return sha256(canonical);
+}
+
+function computeMerkleRoot(hashes: string[]): string {
+  if (hashes.length === 0) return sha256('empty');
+  if (hashes.length === 1) return hashes[0];
+
+  const next: string[] = [];
+  for (let i = 0; i < hashes.length; i += 2) {
+    if (i + 1 < hashes.length) {
+      next.push(sha256(hashes[i] + hashes[i + 1]));
+    } else {
+      next.push(hashes[i]); // odd leaf promoted
+    }
+  }
+  return computeMerkleRoot(next);
+}
+
+// ─── Current Epoch ──────────────────────────
+
+export function currentEpoch(): number {
+  return Math.floor((Date.now() - EPOCH_ZERO) / EPOCH_DURATION_MS);
+}
+
+// ─── MobyDB Class ───────────────────────────
+
+export class MobyDB {
+  private db: Database.Database;
+  private workerPk: string;
+  private h3Cell: string;
+
+  constructor(workerPk: string, h3Cell: string) {
+    this.workerPk = workerPk;
+    this.h3Cell = h3Cell;
+
+    // Ensure directory exists
+    const dir = path.dirname(DB_PATH);
+    if (!fs.existsSync(dir)) {
+      fs.mkdirSync(dir, { recursive: true });
+    }
+
+    // Open SQLite
+    this.db = new Database(DB_PATH);
+    this.db.pragma('journal_mode = WAL');
+    this.db.pragma('synchronous = NORMAL');
+
+    this.initSchema();
+    console.log(`🐋 MobyDB initialized: ${DB_PATH}`);
+    console.log(`   Worker: ${workerPk.substring(0, 16)}...`);
+    console.log(`   Cell: ${h3Cell}`);
+  }
+
+  // ─── Schema ─────────────────────────────
+
+  private initSchema(): void {
+    this.db.exec(`
+      CREATE TABLE IF NOT EXISTS records (
+        id TEXT PRIMARY KEY,
+        h3_cell TEXT NOT NULL,
+        epoch INTEGER NOT NULL,
+        public_key TEXT NOT NULL,
+        collection_type TEXT NOT NULL,
+        payload_type TEXT NOT NULL,
+        data TEXT NOT NULL,
+        signature TEXT,
+        trust_tier TEXT DEFAULT 'Observed',
+        written_at_ms INTEGER NOT NULL,
+        record_hash TEXT NOT NULL,
+        sealed INTEGER DEFAULT 0,
+        epoch_merkle_root TEXT
+      );
+
+      CREATE INDEX IF NOT EXISTS idx_records_address
+        ON records (h3_cell, epoch, public_key);
+      CREATE INDEX IF NOT EXISTS idx_records_collection
+        ON records (collection_type, epoch DESC);
+      CREATE INDEX IF NOT EXISTS idx_records_epoch
+        ON records (epoch, sealed);
+
+      CREATE TABLE IF NOT EXISTS epoch_seals (
+        epoch INTEGER NOT NULL,
+        h3_cell TEXT NOT NULL,
+        worker_pk TEXT NOT NULL,
+        record_count INTEGER NOT NULL,
+        merkle_root TEXT NOT NULL,
+        prev_epoch_hash TEXT NOT NULL,
+        sealed_at_ms INTEGER NOT NULL,
+        signature TEXT,
+        PRIMARY KEY (epoch, h3_cell, worker_pk)
+      );
+
+      CREATE TABLE IF NOT EXISTS sync_queue (
+        id TEXT PRIMARY KEY,
+        record_id TEXT NOT NULL,
+        target TEXT NOT NULL DEFAULT 'central',
+        status TEXT DEFAULT 'pending',
+        attempts INTEGER DEFAULT 0,
+        created_at INTEGER NOT NULL,
+        synced_at INTEGER
+      );
+    `);
+  }
+
+  // ─── Write ──────────────────────────────
+
+  /**
+   * Write a record to local MobyDB.
+   * Returns the record with hash computed.
+   * Typical latency: <0.1ms (SQLite WAL mode).
+   */
+  write(
+    collectionType: CollectionType,
+    payloadType: string,
+    data: Record<string, any>,
+    options?: {
+      h3Cell?: string;
+      epoch?: number;
+      publicKey?: string;
+      trustTier?: string;
+      signature?: string;
+    },
+  ): MobyDBRecord {
+    const epoch = options?.epoch ?? currentEpoch();
+    const id = crypto.randomUUID();
+    const now = Date.now();
+
+    const record: Omit<MobyDBRecord, 'record_hash' | 'sealed' | 'epoch_merkle_root'> = {
+      id,
+      address: {
+        h3_cell: options?.h3Cell ?? this.h3Cell,
+        epoch,
+        public_key: options?.publicKey ?? this.workerPk,
+      },
+      collection_type: collectionType,
+      payload_type: payloadType,
+      data,
+      signature: options?.signature,
+      trust_tier: options?.trustTier ?? 'Observed',
+      written_at_ms: now,
+    };
+
+    const recordHash = computeRecordHash(record);
+
+    const stmt = this.db.prepare(`
+      INSERT INTO records (id, h3_cell, epoch, public_key, collection_type, payload_type, data, signature, trust_tier, written_at_ms, record_hash)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `);
+
+    stmt.run(
+      id,
+      record.address.h3_cell,
+      epoch,
+      record.address.public_key,
+      collectionType,
+      payloadType,
+      JSON.stringify(data),
+      record.signature || null,
+      record.trust_tier,
+      now,
+      recordHash,
+    );
+
+    // Queue for central sync
+    this.queueSync(id);
+
+    return { ...record, record_hash: recordHash, sealed: false };
+  }
+
+  // ─── Convenience Writers ────────────────
+
+  /** Write an inference record after AI compute */
+  writeInference(data: {
+    model: string;
+    provider: string;
+    tokens_in: number;
+    tokens_out: number;
+    latency_ms: number;
+    prompt_hash: string;
+    response_hash: string;
+    requester_pk: string;
+    job_id?: string;
+  }): MobyDBRecord {
+    return this.write('Inference', 'hive/inference', data);
+  }
+
+  /** Write a tile record after rendering/proxying */
+  writeTile(data: {
+    format: string;
+    zoom: number;
+    style: string;
+    tile_hash: string;
+    tile_bytes: number;
+    render_ms: number;
+    source: string;
+    cached: boolean;
+    parent_job?: string;
+  }): MobyDBRecord {
+    return this.write('Tile', data.format === 'mvt' ? 'tile/vector' : 'tile/raster', data);
+  }
+
+  /** Write a compound job record linking multiple steps */
+  writeComputeJob(data: {
+    job_id: string;
+    requester_pk: string;
+    steps: Array<{ id: string; type: string; record_hash: string }>;
+    total_gns: number;
+    stellar_tx?: string;
+    latency_ms: number;
+  }): MobyDBRecord {
+    return this.write('ComputeJob', 'compute/job', data);
+  }
+
+  // ─── Query ──────────────────────────────
+
+  /** Query records near an H3 cell in an epoch range */
+  query(options: {
+    h3Cell?: string;
+    epochStart?: number;
+    epochEnd?: number;
+    collectionType?: CollectionType;
+    limit?: number;
+  }): MobyDBRecord[] {
+    let sql = 'SELECT * FROM records WHERE 1=1';
+    const params: any[] = [];
+
+    if (options.h3Cell) {
+      sql += ' AND h3_cell = ?';
+      params.push(options.h3Cell);
+    }
+    if (options.epochStart !== undefined) {
+      sql += ' AND epoch >= ?';
+      params.push(options.epochStart);
+    }
+    if (options.epochEnd !== undefined) {
+      sql += ' AND epoch <= ?';
+      params.push(options.epochEnd);
+    }
+    if (options.collectionType) {
+      sql += ' AND collection_type = ?';
+      params.push(options.collectionType);
+    }
+    sql += ' ORDER BY written_at_ms DESC LIMIT ?';
+    params.push(options.limit || 100);
+
+    const rows = this.db.prepare(sql).all(...params) as any[];
+    return rows.map(r => ({
+      ...r,
+      address: { h3_cell: r.h3_cell, epoch: r.epoch, public_key: r.public_key },
+      data: JSON.parse(r.data),
+      sealed: !!r.sealed,
+    }));
+  }
+
+  /** Get a single record by ID */
+  get(id: string): MobyDBRecord | null {
+    const r = this.db.prepare('SELECT * FROM records WHERE id = ?').get(id) as any;
+    if (!r) return null;
+    return {
+      ...r,
+      address: { h3_cell: r.h3_cell, epoch: r.epoch, public_key: r.public_key },
+      data: JSON.parse(r.data),
+      sealed: !!r.sealed,
+    };
+  }
+
+  /** Count records by collection type */
+  stats(): Record<string, number> {
+    const rows = this.db.prepare(
+      'SELECT collection_type, count(*) as cnt FROM records GROUP BY collection_type'
+    ).all() as any[];
+    const result: Record<string, number> = {};
+    rows.forEach(r => result[r.collection_type] = r.cnt);
+    return result;
+  }
+
+  // ─── Epoch Sealing ──────────────────────
+
+  /**
+   * Seal an epoch: compute Merkle root of all records,
+   * mark them as sealed, store the epoch seal.
+   */
+  sealEpoch(epoch: number): EpochSeal | null {
+    // Get all unsealed records for this epoch
+    const records = this.db.prepare(
+      'SELECT record_hash FROM records WHERE epoch = ? AND sealed = 0 ORDER BY written_at_ms ASC'
+    ).all(epoch) as any[];
+
+    if (records.length === 0) return null;
+
+    const hashes = records.map(r => r.record_hash);
+    const merkleRoot = computeMerkleRoot(hashes);
+
+    // Get previous epoch seal for chaining
+    const prevSeal = this.db.prepare(
+      'SELECT merkle_root FROM epoch_seals WHERE epoch = ? AND h3_cell = ? AND worker_pk = ?'
+    ).get(epoch - 1, this.h3Cell, this.workerPk) as any;
+    const prevEpochHash = prevSeal?.merkle_root || sha256('genesis');
+
+    const seal: EpochSeal = {
+      epoch,
+      h3_cell: this.h3Cell,
+      worker_pk: this.workerPk,
+      record_count: records.length,
+      merkle_root: merkleRoot,
+      prev_epoch_hash: prevEpochHash,
+      sealed_at_ms: Date.now(),
+    };
+
+    // Transaction: mark records sealed + insert seal
+    const transaction = this.db.transaction(() => {
+      this.db.prepare(
+        'UPDATE records SET sealed = 1, epoch_merkle_root = ? WHERE epoch = ? AND sealed = 0'
+      ).run(merkleRoot, epoch);
+
+      this.db.prepare(`
+        INSERT OR REPLACE INTO epoch_seals (epoch, h3_cell, worker_pk, record_count, merkle_root, prev_epoch_hash, sealed_at_ms, signature)
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+      `).run(seal.epoch, seal.h3_cell, seal.worker_pk, seal.record_count, seal.merkle_root, seal.prev_epoch_hash, seal.sealed_at_ms, seal.signature || null);
+    });
+    transaction();
+
+    console.log(`🔒 Epoch ${epoch} sealed: ${records.length} records, root=${merkleRoot.substring(0, 16)}...`);
+    return seal;
+  }
+
+  /**
+   * Auto-seal previous epochs that have unsealed records.
+   * Call this periodically (e.g. every minute).
+   */
+  autoSeal(): EpochSeal[] {
+    const current = currentEpoch();
+    const unsealed = this.db.prepare(
+      'SELECT DISTINCT epoch FROM records WHERE sealed = 0 AND epoch < ? ORDER BY epoch ASC'
+    ).all(current) as any[];
+
+    const seals: EpochSeal[] = [];
+    for (const row of unsealed) {
+      const seal = this.sealEpoch(row.epoch);
+      if (seal) seals.push(seal);
+    }
+    return seals;
+  }
+
+  // ─── Proof Generation ───────────────────
+
+  /**
+   * Generate a Merkle proof for a specific record.
+   * Proves this record was included in the epoch's Merkle tree.
+   */
+  generateProof(recordId: string): {
+    record_hash: string;
+    epoch: number;
+    merkle_root: string;
+    proof_path: string[];
+    verify_url: string;
+  } | null {
+    const record = this.get(recordId);
+    if (!record || !record.sealed) return null;
+
+    // Get all record hashes in the same epoch (in order)
+    const rows = this.db.prepare(
+      'SELECT record_hash FROM records WHERE epoch = ? AND h3_cell = ? ORDER BY written_at_ms ASC'
+    ).all(record.address.epoch, record.address.h3_cell) as any[];
+
+    const hashes = rows.map(r => r.record_hash);
+    const proofPath = generateMerkleProofPath(hashes, record.record_hash);
+
+    return {
+      record_hash: record.record_hash,
+      epoch: record.address.epoch,
+      merkle_root: record.epoch_merkle_root || '',
+      proof_path: proofPath,
+      verify_url: `https://mobydb.com/proof/${record.address.h3_cell}/${record.address.epoch}/${record.address.public_key}`,
+    };
+  }
+
+  // ─── Sync Queue ─────────────────────────
+
+  private queueSync(recordId: string): void {
+    this.db.prepare(`
+      INSERT OR IGNORE INTO sync_queue (id, record_id, target, created_at)
+      VALUES (?, ?, 'central', ?)
+    `).run(crypto.randomUUID(), recordId, Date.now());
+  }
+
+  /** Get records pending sync to central server */
+  getPendingSync(limit = 50): string[] {
+    const rows = this.db.prepare(
+      "SELECT record_id FROM sync_queue WHERE status = 'pending' ORDER BY created_at ASC LIMIT ?"
+    ).all(limit) as any[];
+    return rows.map(r => r.record_id);
+  }
+
+  /** Mark records as synced */
+  markSynced(recordIds: string[]): void {
+    const stmt = this.db.prepare(
+      "UPDATE sync_queue SET status = 'synced', synced_at = ? WHERE record_id = ?"
+    );
+    const now = Date.now();
+    const transaction = this.db.transaction(() => {
+      for (const id of recordIds) {
+        stmt.run(now, id);
+      }
+    });
+    transaction();
+  }
+
+  // ─── Lifecycle ──────────────────────────
+
+  /** Close the database */
+  close(): void {
+    this.db.close();
+    console.log('🐋 MobyDB closed');
+  }
+
+  /** Get the database file path */
+  get path(): string {
+    return DB_PATH;
+  }
+
+  /** Get total record count */
+  get recordCount(): number {
+    const r = this.db.prepare('SELECT count(*) as cnt FROM records').get() as any;
+    return r.cnt;
+  }
+}
+
+// ─── Merkle Proof Path ──────────────────────
+
+function generateMerkleProofPath(hashes: string[], targetHash: string): string[] {
+  if (hashes.length <= 1) return [];
+
+  const path: string[] = [];
+  let level = [...hashes];
+  let targetIdx = level.indexOf(targetHash);
+
+  while (level.length > 1) {
+    const nextLevel: string[] = [];
+    for (let i = 0; i < level.length; i += 2) {
+      if (i + 1 < level.length) {
+        // If target is at i, sibling is i+1 (right)
+        // If target is at i+1, sibling is i (left)
+        if (i === targetIdx) {
+          path.push('R:' + level[i + 1]);
+          targetIdx = Math.floor(i / 2);
+        } else if (i + 1 === targetIdx) {
+          path.push('L:' + level[i]);
+          targetIdx = Math.floor(i / 2);
+        }
+        nextLevel.push(sha256(level[i] + level[i + 1]));
+      } else {
+        if (i === targetIdx) {
+          targetIdx = Math.floor(i / 2);
+        }
+        nextLevel.push(level[i]);
+      }
+    }
+    level = nextLevel;
+  }
+
+  return path;
+}
+
+// ─── Singleton ──────────────────────────────
+
+let _instance: MobyDB | null = null;
+
+export function initMobyDB(workerPk: string, h3Cell: string): MobyDB {
+  if (_instance) return _instance;
+  _instance = new MobyDB(workerPk, h3Cell);
+  return _instance;
+}
+
+export function getMobyDB(): MobyDB | null {
+  return _instance;
+}