@spooky-sync/core 0.0.0-canary.1

package/src/services/database/local-migrator.ts

@@ -0,0 +1,203 @@
+import type { Surreal } from 'surrealdb';
+import { Logger, createLogger } from '../logger/index';
+import { LocalDatabaseService } from './local';
+
+export interface SchemaRecord {
+  hash: string;
+  created_at: string;
+}
+
+export const sha1 = async (str: string): Promise<string> => {
+  const enc = new TextEncoder();
+  const hash = await crypto.subtle.digest('SHA-1', enc.encode(str));
+  return Array.from(new Uint8Array(hash))
+    .map((v) => v.toString(16).padStart(2, '0'))
+    .join('');
+};
+
+export class LocalMigrator {
+  private logger: Logger;
+
+  constructor(
+    private localDb: LocalDatabaseService,
+    logger: Logger
+  ) {
+    this.logger = logger.child({ service: 'LocalMigrator' });
+    logger?.child({ service: 'LocalMigrator' }) ??
+      createLogger('info').child({ service: 'LocalMigrator' });
+  }
+
+  async provision(schemaSurql: string): Promise<void> {
+    const hash = await sha1(schemaSurql);
+
+    const { database } = this.localDb.getConfig();
+
+    if (await this.isSchemaUpToDate(hash)) {
+      this.logger.info(
+        { Category: 'spooky-client::LocalMigrator::provision' },
+        '[Provisioning] Schema is up to date, skipping migration'
+      );
+      return;
+    }
+
+    await this.recreateDatabase(database);
+
+    const systemSchema = `
+      DEFINE TABLE IF NOT EXISTS _spooky_stream_processor_state SCHEMALESS PERMISSIONS FOR select, create, update, delete WHERE true;
+      DEFINE TABLE IF NOT EXISTS _spooky_query SCHEMALESS PERMISSIONS FOR select, create, update, delete WHERE true;
+      DEFINE TABLE IF NOT EXISTS _spooky_schema SCHEMALESS PERMISSIONS FOR select, create, update, delete WHERE true;
+      DEFINE TABLE IF NOT EXISTS _spooky_pending_mutations SCHEMALESS PERMISSIONS FOR select, create, update, delete WHERE true;
+    `;
+    const fullSchema = schemaSurql + '\n' + systemSchema;
+
+    const statements = this.splitStatements(fullSchema);
+
+    for (let i = 0; i < statements.length; i++) {
+      const statement = statements[i];
+      // Strip comments to check if it's an index definition
+      const cleanStatement = statement.replace(/--.*/g, '').trim();
+
+      // SKIP INDEXES: WASM engine hangs on DEFINE INDEX (confirmed)
+      if (cleanStatement.toUpperCase().startsWith('DEFINE INDEX')) {
+        this.logger.warn(
+          { Category: 'spooky-client::LocalMigrator::provision' },
+          `[Provisioning] Skipping index definition (WASM hang avoidance): ${cleanStatement.substring(0, 50)}...`
+        );
+        continue;
+      }
+
+      try {
+        this.logger.info(
+          { Category: 'spooky-client::LocalMigrator::provision' },
+          `[Provisioning] (${i + 1}/${statements.length}) Executing: ${statement.substring(0, 50)}...`
+        );
+        await this.localDb.query(statement);
+        this.logger.info(
+          { Category: 'spooky-client::LocalMigrator::provision' },
+          `[Provisioning] (${i + 1}/${statements.length}) Done`
+        );
+      } catch (e) {
+        this.logger.error(
+          { Category: 'spooky-client::LocalMigrator::provision' },
+          `[Provisioning] (${i + 1}/${statements.length}) Error executing statement: ${statement}`
+        );
+        throw e;
+      }
+    }
+
+    await this.createHashRecord(hash);
+  }
+
+  private async isSchemaUpToDate(hash: string): Promise<boolean> {
+    try {
+      const [lastSchemaRecord] = await this.localDb.query<any>(
+        `SELECT hash, created_at FROM ONLY _spooky_schema ORDER BY created_at DESC LIMIT 1;`
+      );
+      return lastSchemaRecord?.hash === hash;
+    } catch (error) {
+      return false;
+    }
+  }
+
+  private async recreateDatabase(database: string) {
+    try {
+      await this.localDb.query(`DEFINE DATABASE _spooky_temp;`);
+    } catch (e) {
+      // Ignore if exists
+    }
+
+    try {
+      await this.localDb.query(`
+        USE DB _spooky_temp;
+        REMOVE DATABASE ${database};
+      `);
+    } catch (e) {
+      // Ignore error if database doesn't exist
+    }
+
+    await this.localDb.query(`
+      DEFINE DATABASE ${database};
+      USE DB ${database};
+    `);
+  }
+
+  private splitStatements(schema: string): string[] {
+    const statements: string[] = [];
+    let current = '';
+    let depth = 0;
+    let inQuote = false;
+    let quoteChar = '';
+    let inComment = false;
+
+    for (let i = 0; i < schema.length; i++) {
+      const char = schema[i];
+      const nextChar = schema[i + 1];
+
+      // Handle Comments
+      if (inComment) {
+        current += char;
+        if (char === '\n') {
+          inComment = false;
+        }
+        continue;
+      }
+
+      // Start of comment
+      if (!inQuote && char === '-' && nextChar === '-') {
+        inComment = true;
+        current += char;
+        continue;
+      }
+
+      if (inQuote) {
+        current += char;
+        if (char === quoteChar && schema[i - 1] !== '\\') {
+          inQuote = false;
+        }
+        continue;
+      }
+
+      if (char === '"' || char === "'") {
+        inQuote = true;
+        quoteChar = char;
+        current += char;
+        continue;
+      }
+
+      if (char === '{') {
+        depth++;
+        current += char;
+        continue;
+      }
+
+      if (char === '}') {
+        depth--;
+        current += char;
+        continue;
+      }
+
+      if (char === ';' && depth === 0) {
+        if (current.trim().length > 0) {
+          statements.push(current.trim());
+        }
+        current = '';
+        continue;
+      }
+
+      current += char;
+    }
+
+    if (current.trim().length > 0) {
+      statements.push(current.trim());
+    }
+
+    return statements;
+  }
+
+  private async createHashRecord(hash: string) {
+    await this.localDb.query(
+      `UPSERT _spooky_schema SET hash = $hash, created_at = time::now() WHERE hash = $hash;`,
+      { hash }
+    );
+  }
+}

package/src/services/database/local.ts

@@ -0,0 +1,99 @@
+import { applyDiagnostics, DateTime, Diagnostic, RecordId, Surreal } from 'surrealdb';
+import { createWasmWorkerEngines } from '@surrealdb/wasm';
+import { SpookyConfig } from '../../types';
+import { Logger } from '../logger/index';
+import { AbstractDatabaseService } from './database';
+import { createDatabaseEventSystem, DatabaseEventTypes } from './events/index';
+import { encodeRecordId, parseRecordIdString, surql } from '../../utils/index';
+
+export class LocalDatabaseService extends AbstractDatabaseService {
+  private config: SpookyConfig<any>['database'];
+  protected eventType = DatabaseEventTypes.LocalQuery;
+
+  constructor(config: SpookyConfig<any>['database'], logger: Logger) {
+    const events = createDatabaseEventSystem();
+    super(
+      new Surreal({
+        codecOptions: {
+          valueDecodeVisitor(value) {
+            if (value instanceof RecordId) {
+              return encodeRecordId(value);
+            }
+
+            if (value instanceof DateTime) {
+              return value.toDate();
+            }
+
+            return value;
+          },
+        },
+        engines: applyDiagnostics(
+          createWasmWorkerEngines(),
+          ({ key, type, phase, ...other }: Diagnostic) => {
+            if (phase === 'progress' || phase === 'after') {
+              logger.trace(
+                {
+                  ...other,
+                  key,
+                  type,
+                  phase,
+                  service: 'surrealdb:local',
+                  Category: 'spooky-client::LocalDatabaseService::diagnostics',
+                },
+                `Local SurrealDB diagnostics captured ${type}:${phase}`
+              );
+            }
+          }
+        ),
+      }),
+      logger,
+      events
+    );
+    this.config = config;
+  }
+
+  getConfig(): SpookyConfig<any>['database'] {
+    return this.config;
+  }
+
+  async connect(): Promise<void> {
+    const { namespace, database } = this.getConfig();
+    this.logger.info(
+      { namespace, database, Category: 'spooky-client::LocalDatabaseService::connect' },
+      'Connecting to local database'
+    );
+    try {
+      const store = this.getConfig().store ?? 'memory';
+      const storeUrl = store === 'memory' ? 'mem://' : 'indxdb://spooky';
+      this.logger.debug(
+        { storeUrl, Category: 'spooky-client::LocalDatabaseService::connect' },
+        '[LocalDatabaseService] Calling client.connect'
+      );
+      await this.client.connect(storeUrl, {});
+      this.logger.debug(
+        { namespace, database, Category: 'spooky-client::LocalDatabaseService::connect' },
+        '[LocalDatabaseService] client.connect returned. Calling client.use'
+      );
+
+      await this.client.use({
+        namespace,
+        database,
+      });
+      this.logger.debug(
+        { Category: 'spooky-client::LocalDatabaseService::connect' },
+        '[LocalDatabaseService] client.use returned'
+      );
+
+      this.logger.info(
+        { Category: 'spooky-client::LocalDatabaseService::connect' },
+        'Connected to local database'
+      );
+    } catch (err) {
+      this.logger.error(
+        { err, Category: 'spooky-client::LocalDatabaseService::connect' },
+        'Failed to connect to local database'
+      );
+      throw err;
+    }
+  }
+}

package/src/services/database/remote.ts

@@ -0,0 +1,110 @@
+import {
+  applyDiagnostics,
+  createRemoteEngines,
+  Diagnostic,
+  Surreal,
+  SurrealTransaction,
+} from 'surrealdb';
+import { SpookyConfig } from '../../types';
+import { Logger } from '../logger/index';
+import { AbstractDatabaseService } from './database';
+import { createDatabaseEventSystem, DatabaseEventTypes } from './events/index';
+
+export class RemoteDatabaseService extends AbstractDatabaseService {
+  private config: SpookyConfig<any>['database'];
+  protected eventType = DatabaseEventTypes.RemoteQuery;
+
+  constructor(config: SpookyConfig<any>['database'], logger: Logger) {
+    const events = createDatabaseEventSystem();
+    super(
+      new Surreal({
+        engines: applyDiagnostics(
+          createRemoteEngines(),
+          ({ key, type, phase, ...other }: Diagnostic) => {
+            if (phase === 'progress' || phase === 'after') {
+              logger.trace(
+                {
+                  ...other,
+                  key,
+                  type,
+                  phase,
+                  service: 'surrealdb:remote',
+                  Category: 'spooky-client::RemoteDatabaseService::diagnostics',
+                },
+                `Remote SurrealDB diagnostics captured ${type}:${phase}`
+              );
+            }
+          }
+        ),
+      }),
+      logger,
+      events
+    );
+    this.config = config;
+  }
+
+  getConfig(): SpookyConfig<any>['database'] {
+    return this.config;
+  }
+
+  async connect(): Promise<void> {
+    const { endpoint, token, namespace, database } = this.getConfig();
+    if (endpoint) {
+      this.logger.info(
+        {
+          endpoint,
+          namespace,
+          database,
+          Category: 'spooky-client::RemoteDatabaseService::connect',
+        },
+        'Connecting to remote database'
+      );
+      try {
+        await this.client.connect(endpoint);
+        await this.client.use({
+          namespace,
+          database,
+        });
+
+        if (token) {
+          this.logger.debug(
+            { Category: 'spooky-client::RemoteDatabaseService::connect' },
+            'Authenticating with token'
+          );
+          await this.client.authenticate(token);
+        }
+        this.logger.info(
+          { Category: 'spooky-client::RemoteDatabaseService::connect' },
+          'Connected to remote database'
+        );
+      } catch (err) {
+        this.logger.error(
+          { err, Category: 'spooky-client::RemoteDatabaseService::connect' },
+          'Failed to connect to remote database'
+        );
+        throw err;
+      }
+    } else {
+      this.logger.warn(
+        { Category: 'spooky-client::RemoteDatabaseService::connect' },
+        'No endpoint configured for remote database'
+      );
+    }
+  }
+
+  async signin(params: any): Promise<any> {
+    return this.client.signin(params);
+  }
+
+  async signup(params: any): Promise<any> {
+    return this.client.signup(params);
+  }
+
+  async authenticate(token: string): Promise<any> {
+    return this.client.authenticate(token);
+  }
+
+  async invalidate(): Promise<void> {
+    return this.client.invalidate();
+  }
+}

package/src/services/logger/index.ts

@@ -0,0 +1,118 @@
+import pino, { Level, type Logger as PinoLogger, type LoggerOptions } from 'pino';
+import { LoggerProvider, BatchLogRecordProcessor } from '@opentelemetry/sdk-logs';
+import { OTLPLogExporter } from '@opentelemetry/exporter-logs-otlp-proto';
+import { resourceFromAttributes } from '@opentelemetry/resources';
+import { ATTR_SERVICE_NAME } from '@opentelemetry/semantic-conventions';
+import { createContextKey } from '@opentelemetry/api';
+
+const CATEGORY_KEY = createContextKey('Category');
+
+export type Logger = PinoLogger;
+
+// Map pino levels to OTEL severity numbers
+// https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/logs/data-model.md#severity-fields
+function mapLevelToSeverityNumber(level: string): number {
+  switch (level) {
+    case 'trace':
+      return 1;
+    case 'debug':
+      return 5;
+    case 'info':
+      return 9;
+    case 'warn':
+      return 13;
+    case 'error':
+      return 17;
+    case 'fatal':
+      return 21;
+    default:
+      return 9;
+  }
+}
+
+export function createLogger(level: Level = 'info', otelEndpoint?: string): Logger {
+  const browserConfig: LoggerOptions['browser'] = {
+    asObject: true,
+    write: (o: any) => {
+      console.log(JSON.stringify(o));
+    },
+  };
+
+  if (otelEndpoint) {
+    // Initialize OTEL LoggerProvider
+    const resource = resourceFromAttributes({
+      [ATTR_SERVICE_NAME]: 'spooky-client',
+    });
+
+    const exporter = new OTLPLogExporter({
+      url: otelEndpoint,
+    });
+
+    // Pass processors in constructor as this SDK version requires it
+    const loggerProvider = new LoggerProvider({
+      resource,
+      processors: [new BatchLogRecordProcessor(exporter)],
+    });
+
+    const otelLogger: Record<string, ReturnType<typeof loggerProvider.getLogger>> = {};
+
+    const getOtelLogger = (category: string) => {
+      if (!otelLogger[category]) {
+        otelLogger[category] = loggerProvider.getLogger(category);
+      }
+      return otelLogger[category];
+    };
+
+    browserConfig.transmit = {
+      level: level,
+      send: (levelLabel: string, logEvent: any) => {
+        try {
+          const messages = [...logEvent.messages];
+          const severityNumber = mapLevelToSeverityNumber(levelLabel);
+
+          // Construct the message body
+          let body = '';
+          const msg = messages.pop();
+
+          if (typeof msg === 'string') {
+            body = msg;
+          } else if (msg) {
+            body = JSON.stringify(msg);
+          }
+
+          let category = 'spooky-client::unknown';
+
+          const attributes = {};
+          for (const msg of messages) {
+            if (typeof msg === 'object') {
+              if (msg.Category) {
+                category = msg.Category;
+                delete msg.Category;
+              }
+              Object.assign(attributes, msg);
+            }
+          }
+
+          // Emit to OTEL SDK
+          getOtelLogger(category).emit({
+            severityNumber: severityNumber,
+            severityText: levelLabel.toUpperCase(),
+            body: body,
+            attributes: {
+              ...logEvent.bindings[0],
+              ...attributes,
+            },
+            timestamp: new Date(logEvent.ts),
+          });
+        } catch (e) {
+          console.warn('Failed to transmit log to OTEL endpoint', e);
+        }
+      },
+    };
+  }
+
+  return pino({
+    level,
+    browser: browserConfig,
+  });
+}

package/src/services/persistence/localstorage.ts

@@ -0,0 +1,26 @@
+import { Logger } from 'pino';
+import { PersistenceClient } from '../../types';
+
+export class LocalStoragePersistenceClient implements PersistenceClient {
+  private logger: Logger;
+
+  constructor(logger: Logger) {
+    this.logger = logger.child({ service: 'PersistenceClient:LocalStorage' });
+  }
+
+  set<T>(key: string, value: T): Promise<void> {
+    localStorage.setItem(key, JSON.stringify(value));
+    return Promise.resolve();
+  }
+
+  get<T>(key: string): Promise<T | null> {
+    const value = localStorage.getItem(key);
+    if (!value) return Promise.resolve(null);
+    return Promise.resolve(JSON.parse(value));
+  }
+
+  remove(key: string): Promise<void> {
+    localStorage.removeItem(key);
+    return Promise.resolve();
+  }
+}

package/src/services/persistence/surrealdb.ts

@@ -0,0 +1,62 @@
+import { PersistenceClient } from '../../types';
+import { parseRecordIdString, surql } from '../../utils/index';
+import { Logger } from 'pino';
+import { AbstractDatabaseService } from '../database/database';
+
+export class SurrealDBPersistenceClient implements PersistenceClient {
+  private logger: Logger;
+
+  constructor(
+    private db: AbstractDatabaseService,
+    logger: Logger
+  ) {
+    this.logger = logger.child({ service: 'PersistenceClient:SurrealDb' });
+  }
+
+  async set<T>(key: string, val: T) {
+    try {
+      const id = parseRecordIdString(`_spooky_kv:${key}`);
+      await this.db.query(surql.seal(surql.upsert('id', 'data')), { id, data: { val } });
+    } catch (error) {
+      this.logger.error(
+        { error, Category: 'spooky-client::SurrealDBPersistenceClient::set' },
+        'Failed to set KV'
+      );
+      throw error;
+    }
+  }
+
+  async get<T>(key: string) {
+    try {
+      const id = parseRecordIdString(`_spooky_kv:${key}`);
+      const [result] = await this.db.query<[{ val: T }]>(
+        surql.seal(surql.selectById('id', ['val'])),
+        {
+          id,
+        }
+      );
+      if (!result?.val) {
+        return null;
+      }
+      return result.val;
+    } catch (error) {
+      this.logger.warn(
+        { error, Category: 'spooky-client::SurrealDBPersistenceClient::get' },
+        'Failed to get KV'
+      );
+      return null;
+    }
+  }
+
+  async remove(key: string) {
+    try {
+      const id = parseRecordIdString(`_spooky_kv:${key}`);
+      await this.db.query(surql.seal(surql.delete('id')), { id });
+    } catch (err) {
+      this.logger.info(
+        { err, Category: 'spooky-client::SurrealDBPersistenceClient::remove' },
+        'Failed to delete KV'
+      );
+    }
+  }
+}