@spooky-sync/core 0.0.0-canary.1

package/README.md

@@ -0,0 +1,21 @@
+## Register Query Flow
+
+1. Query registering (local) - Data Module
+2. Query get local data - Data Module
+3. (Query registing (remote) - Sync Module)
+
+## Query Local Updates
+
+1. Local Mutation is executed - Data Module
+2. Mutation is enqueued for remote - Sync Moduke
+3. New Record is ingested to DBSP - Data Module
+4. Related Queries are updated - Data Module
+5. (Sync Module syncs the mutation to remote - Sync Module)
+
+## Query Live Updates
+
+1. Live Update is received - Sync Module
+2. Version is compared with local version - Sync Module
+3. If remote version is higher, the record is updated - Sync Module
+4. Record is ingested to DBSP Module - Data Module
+5. Related Queries are updated - Data Module

package/dist/index.d.ts

@@ -0,0 +1,590 @@
+import * as surrealdb0 from "surrealdb";
+import { Duration, RecordId, Surreal, SurrealTransaction } from "surrealdb";
+import { AccessDefinition, BackendNames, BackendRoutes, ColumnSchema, GetTable, QueryBuilder, QueryOptions, RecordId as RecordId$1, RoutePayload, SchemaStructure, TableModel, TableNames, TypeNameToTypeMap } from "@spooky-sync/query-builder";
+import { Level, Level as Level$1, Logger } from "pino";
+
+//#region src/events/index.d.ts
+/**
+ * Utility type to define the payload structure of an event.
+ * If the payload type P is never, it defines payload as undefined.
+ */
+type EventPayloadDefinition<P> = [P] extends [never] ? {
+  payload: undefined;
+} : {
+  payload: P;
+};
+/**
+ * Defines the structure of an event with a specific type and payload.
+ * @template T The string literal type of the event.
+ * @template P The type of the event payload.
+ */
+type EventDefinition<T extends string, P> = {
+  type: T;
+} & EventPayloadDefinition<P>;
+/**
+ * A map of event types to their definitions.
+ * Keys are event names, values are EventDefinitions.
+ */
+type EventTypeMap = Record<string, EventDefinition<any, unknown> | EventDefinition<any, never>>;
+/**
+ * Options for pushing/emitting events.
+ */
+interface PushEventOptions {
+  /** Configuration for debouncing the event. */
+  debounced?: {
+    key: string;
+    delay: number;
+  };
+}
+/**
+ * Extracts the full Event object type from the map for a given key.
+ */
+type Event<E extends EventTypeMap, T extends EventType<E>> = E[T];
+/**
+ * Extracts the payload type from the map for a given key.
+ */
+type EventPayload<E extends EventTypeMap, T extends EventType<E>> = E[T]['payload'];
+/**
+ * Array of available event type keys.
+ */
+type EventTypes<E extends EventTypeMap> = (keyof E)[];
+/**
+ * Represents a valid key (event name) from the EventTypeMap.
+ */
+type EventType<E extends EventTypeMap> = keyof E;
+/**
+ * Function signature for an event handler.
+ */
+type EventHandler<E extends EventTypeMap, T extends EventType<E>> = (event: Event<E, T>) => void;
+/**
+ * Options when subscribing to an event.
+ */
+type EventSubscriptionOptions$1 = {
+  /** If true, the handler will be called immediately with the last emitted event of this type (if any). */
+  immediately?: boolean;
+  /** If true, the subscription will be automatically removed after the first event is handled. */
+  once?: boolean;
+};
+/**
+ * A type-safe event system that handles subscription, emission (including debouncing), and buffering of events.
+ * @template E The EventTypeMap defining all supported events.
+ */
+declare class EventSystem<E extends EventTypeMap> {
+  private _eventTypes;
+  private subscriberId;
+  private isProcessing;
+  private buffer;
+  private subscribers;
+  private subscribersTypeMap;
+  private lastEvents;
+  private debouncedEvents;
+  constructor(_eventTypes: EventTypes<E>);
+  get eventTypes(): EventTypes<E>;
+  /**
+   * Subscribes a handler to a specific event type.
+   * @param type The event type to subscribe to.
+   * @param handler The function to call when the event occurs.
+   * @param options Subscription options (once, immediately).
+   * @returns A subscription ID that can be used to unsubscribe.
+   */
+  subscribe<T extends EventType<E>>(type: T, handler: EventHandler<E, T>, options?: EventSubscriptionOptions$1): number;
+  /**
+   * Subscribes a handler to multiple event types.
+   * @param types An array of event types to subscribe to.
+   * @param handler The function to call when any of the events occur.
+   * @param options Subscription options.
+   * @returns An array of subscription IDs.
+   */
+  subscribeMany<T extends EventType<E>>(types: T[], handler: EventHandler<E, T>, options?: EventSubscriptionOptions$1): number[];
+  /**
+   * Unsubscribes a specific subscription by ID.
+   * @param id The subscription ID returned by subscribe().
+   * @returns True if the subscription was found and removed, false otherwise.
+   */
+  unsubscribe(id: number): boolean;
+  /**
+   * Emits an event with the given type and payload.
+   * @param type The type of event to emit.
+   * @param payload The data associated with the event.
+   */
+  emit<T extends EventType<E>, P extends EventPayload<E, T>>(type: T, payload: P): void;
+  /**
+   * Adds a fully constructed event object to the system.
+   * Similar to emit, but takes the full event object directly.
+   * Supports debouncing if options are provided.
+   * @param event The event object.
+   * @param options Options for the event push (e.g., debouncing).
+   */
+  addEvent<T extends EventType<E>>(event: Event<E, T>, options?: PushEventOptions): void;
+  private handleDebouncedEvent;
+  private scheduleProcessing;
+  private processEvents;
+  private dequeue;
+  private setLastEvent;
+  private broadcastEvent;
+}
+//#endregion
+//#region src/services/logger/index.d.ts
+type Logger$1 = Logger;
+//#endregion
+//#region src/services/database/events/index.d.ts
+declare const DatabaseEventTypes: {
+  readonly LocalQuery: "DATABASE_LOCAL_QUERY";
+  readonly RemoteQuery: "DATABASE_REMOTE_QUERY";
+};
+interface DatabaseQueryEventPayload {
+  query: string;
+  vars?: Record<string, unknown>;
+  duration: number;
+  success: boolean;
+  error?: string;
+  timestamp: number;
+}
+type DatabaseEventTypeMap = {
+  [DatabaseEventTypes.LocalQuery]: EventDefinition<typeof DatabaseEventTypes.LocalQuery, DatabaseQueryEventPayload>;
+  [DatabaseEventTypes.RemoteQuery]: EventDefinition<typeof DatabaseEventTypes.RemoteQuery, DatabaseQueryEventPayload>;
+};
+type DatabaseEventSystem = EventSystem<DatabaseEventTypeMap>;
+//#endregion
+//#region src/utils/surql.d.ts
+interface SealedQuery<T = void> {
+  readonly sql: string;
+  readonly extract: (results: unknown[]) => T;
+}
+//#endregion
+//#region src/services/database/database.d.ts
+declare abstract class AbstractDatabaseService {
+  protected client: Surreal;
+  protected logger: Logger$1;
+  protected events: DatabaseEventSystem;
+  protected abstract eventType: typeof DatabaseEventTypes.LocalQuery | typeof DatabaseEventTypes.RemoteQuery;
+  constructor(client: Surreal, logger: Logger$1, events: DatabaseEventSystem);
+  abstract connect(): Promise<void>;
+  getClient(): Surreal;
+  getEvents(): DatabaseEventSystem;
+  tx(): Promise<SurrealTransaction>;
+  private queryQueue;
+  /**
+   * Execute a query with serialized execution to prevent WASM transaction issues.
+   */
+  query<T extends unknown[]>(query: string, vars?: Record<string, unknown>): Promise<T>;
+  execute<T>(query: SealedQuery<T>, vars?: Record<string, unknown>): Promise<T>;
+  close(): Promise<void>;
+}
+//#endregion
+//#region src/services/database/local.d.ts
+declare class LocalDatabaseService extends AbstractDatabaseService {
+  private config;
+  protected eventType: "DATABASE_LOCAL_QUERY";
+  constructor(config: SpookyConfig<any>['database'], logger: Logger$1);
+  getConfig(): SpookyConfig<any>['database'];
+  connect(): Promise<void>;
+}
+//#endregion
+//#region src/services/database/remote.d.ts
+declare class RemoteDatabaseService extends AbstractDatabaseService {
+  private config;
+  protected eventType: "DATABASE_REMOTE_QUERY";
+  constructor(config: SpookyConfig<any>['database'], logger: Logger$1);
+  getConfig(): SpookyConfig<any>['database'];
+  connect(): Promise<void>;
+  signin(params: any): Promise<any>;
+  signup(params: any): Promise<any>;
+  authenticate(token: string): Promise<any>;
+  invalidate(): Promise<void>;
+}
+//#endregion
+//#region src/modules/sync/queue/queue-up.d.ts
+type CreateEvent = {
+  type: 'create';
+  mutation_id: RecordId;
+  record_id: RecordId;
+  data: Record<string, unknown>;
+  record?: Record<string, unknown>;
+  tableName?: string;
+  options?: PushEventOptions;
+};
+type UpdateEvent = {
+  type: 'update';
+  mutation_id: RecordId;
+  record_id: RecordId;
+  data: Record<string, unknown>;
+  record?: Record<string, unknown>;
+  beforeRecord?: Record<string, unknown>;
+  options?: PushEventOptions;
+};
+type DeleteEvent = {
+  type: 'delete';
+  mutation_id: RecordId;
+  record_id: RecordId;
+  options?: PushEventOptions;
+};
+type UpEvent = CreateEvent | UpdateEvent | DeleteEvent;
+//#endregion
+//#region src/services/stream-processor/wasm-types.d.ts
+interface WasmStreamUpdate {
+  query_id: string;
+  result_hash: string;
+  result_data: RecordVersionArray;
+}
+//#endregion
+//#region src/services/stream-processor/index.d.ts
+interface QueryPlanConfig {
+  queryHash: string;
+  surql: string;
+  params: Record<string, any>;
+  ttl: QueryTimeToLive | Duration;
+  lastActiveAt: Date;
+  localArray: RecordVersionArray;
+  remoteArray: RecordVersionArray;
+  meta: {
+    tableName: string;
+    involvedTables?: string[];
+  };
+}
+interface StreamUpdate {
+  queryHash: string;
+  localArray: RecordVersionArray;
+  op?: 'CREATE' | 'UPDATE' | 'DELETE';
+}
+type StreamProcessorEvents = {
+  stream_update: EventDefinition<'stream_update', StreamUpdate[]>;
+};
+/**
+ * Interface for receiving stream updates directly.
+ * Implemented by DataManager and DevToolsService for direct coupling.
+ */
+interface StreamUpdateReceiver {
+  onStreamUpdate(update: StreamUpdate): void;
+}
+declare class StreamProcessorService {
+  events: EventSystem<StreamProcessorEvents>;
+  private db;
+  private persistenceClient;
+  private logger;
+  private processor;
+  private isInitialized;
+  private receivers;
+  constructor(events: EventSystem<StreamProcessorEvents>, db: LocalDatabaseService, persistenceClient: PersistenceClient, logger: Logger);
+  /**
+   * Add a receiver for stream updates.
+   * Multiple receivers can be registered (DataManager, DevTools, etc.)
+   */
+  addReceiver(receiver: StreamUpdateReceiver): void;
+  private notifyUpdates;
+  /**
+   * Initialize the WASM module and processor.
+   * This must be called before using other methods.
+   */
+  init(): Promise<void>;
+  loadState(): Promise<void>;
+  saveState(): Promise<void>;
+  /**
+   * Ingest a record change into the processor.
+   * Emits 'stream_update' event if materialized views are affected.
+   * @param isOptimistic true = local mutation (increment versions), false = remote sync (keep versions)
+   */
+  ingest(table: string, op: 'CREATE' | 'UPDATE' | 'DELETE', id: string, record: any): WasmStreamUpdate[];
+  /**
+   * Register a new query plan.
+   * Emits 'stream_update' with the initial result.
+   */
+  registerQueryPlan(queryPlan: QueryPlanConfig): StreamUpdate | undefined;
+  /**
+   * Unregister a query plan by ID.
+   */
+  unregisterQueryPlan(queryHash: string): void;
+  private normalizeValue;
+}
+//#endregion
+//#region src/types.d.ts
+/**
+ * The type of storage backend to use for the local database.
+ * - 'memory': In-memory storage (transient).
+ * - 'indexeddb': IndexedDB storage (persistent).
+ */
+type StoreType = 'memory' | 'indexeddb';
+/**
+ * Interface for a custom persistence client.
+ * Allows providing a custom storage mechanism for the local database.
+ */
+interface PersistenceClient {
+  /**
+   * Sets a value in the storage.
+   * @param key The key to set.
+   * @param value The value to store.
+   */
+  set<T>(key: string, value: T): Promise<void>;
+  /**
+   * Gets a value from the storage.
+   * @param key The key to retrieve.
+   * @returns The stored value or null if not found.
+   */
+  get<T>(key: string): Promise<T | null>;
+  /**
+   * Removes a value from the storage.
+   * @param key The key to remove.
+   */
+  remove(key: string): Promise<void>;
+}
+/**
+ * Supported Time-To-Live (TTL) values for cached queries.
+ * Format: number + unit (m=minutes, h=hours, d=days).
+ */
+type QueryTimeToLive = '1m' | '5m' | '10m' | '15m' | '20m' | '25m' | '30m' | '1h' | '2h' | '3h' | '4h' | '5h' | '6h' | '7h' | '8h' | '9h' | '10h' | '11h' | '12h' | '1d';
+/**
+ * Result object returned when a query is registered or executed.
+ */
+interface SpookyQueryResult {
+  /** The unique hash identifier for the query. */
+  hash: string;
+}
+type SpookyQueryResultPromise = Promise<SpookyQueryResult>;
+interface EventSubscriptionOptions {
+  priority?: number;
+}
+/**
+ * Configuration options for the Spooky client.
+ * @template S The schema structure type.
+ */
+interface SpookyConfig<S extends SchemaStructure> {
+  /** Database connection configuration. */
+  database: {
+    /** The SurrealDB endpoint URL. */
+    endpoint?: string;
+    /** The namespace to use. */
+    namespace: string;
+    /** The database name. */
+    database: string;
+    /** The local store type implementation. */
+    store?: StoreType;
+    /** Authentication token. */
+    token?: string;
+  };
+  /** Unique client identifier. If not provided, one will be generated. */
+  clientId?: string;
+  /** The schema definition. */
+  schema: S;
+  /** The compiled SURQL schema string. */
+  schemaSurql: string;
+  /** Logging level. */
+  logLevel: Level$1;
+  /**
+   * Persistence client to use.
+   * Can be a custom implementation, 'surrealdb' (default), or 'localstorage'.
+   */
+  persistenceClient?: PersistenceClient | 'surrealdb' | 'localstorage';
+  /** OpenTelemetry collector endpoint for telemetry data. */
+  otelEndpoint?: string;
+  /**
+   * Debounce time in milliseconds for stream updates.
+   * Defaults to 100ms.
+   */
+  streamDebounceTime?: number;
+}
+type QueryHash = string;
+type RecordVersionArray = Array<[string, number]>;
+/**
+ * Represents the difference between two record version sets.
+ * Used for synchronizing local and remote states.
+ */
+interface RecordVersionDiff {
+  /** List of records added. */
+  added: Array<{
+    id: RecordId$1<string>;
+    version: number;
+  }>;
+  /** List of records updated. */
+  updated: Array<{
+    id: RecordId$1<string>;
+    version: number;
+  }>;
+  /** List of record IDs removed. */
+  removed: RecordId$1<string>[];
+}
+/**
+ * Configuration for a specific query instance.
+ * Stores metadata about the query's state, parameters, and versioning.
+ */
+interface QueryConfig {
+  /** The unique ID of the query config record. */
+  id: RecordId$1<string>;
+  /** The SURQL query string. */
+  surql: string;
+  /** Parameters used in the query. */
+  params: Record<string, any>;
+  /** The version array representing the local state of results. */
+  localArray: RecordVersionArray;
+  /** The version array representing the remote (server) state of results. */
+  remoteArray: RecordVersionArray;
+  /** Time-To-Live for this query. */
+  ttl: QueryTimeToLive;
+  /** Timestamp when the query was last accessed/active. */
+  lastActiveAt: Date;
+  /** The name of the table this query targets (if applicable). */
+  tableName: string;
+}
+type QueryConfigRecord = QueryConfig & {
+  id: string;
+};
+/**
+ * Internal state of a live query.
+ */
+interface QueryState {
+  /** The configuration for this query. */
+  config: QueryConfig;
+  /** The current cached records for this query. */
+  records: Record<string, any>[];
+  /** Timer for TTL expiration. */
+  ttlTimer: NodeJS.Timeout | null;
+  /** TTL duration in milliseconds. */
+  ttlDurationMs: number;
+  /** Number of times the query has been updated. */
+  updateCount: number;
+}
+type QueryUpdateCallback = (records: Record<string, any>[]) => void;
+type MutationCallback = (mutations: UpEvent[]) => void;
+type MutationEventType = 'create' | 'update' | 'delete';
+/**
+ * Represents a mutation event (create, update, delete) to be synchronized.
+ */
+interface MutationEvent {
+  /** Example: 'create', 'update', or 'delete'. */
+  type: MutationEventType;
+  /** unique id of the mutation */
+  mutation_id: RecordId$1<string>;
+  /** The ID of the record being mutated. */
+  record_id: RecordId$1<string>;
+  /** The data payload for create/update operations. */
+  data?: any;
+  /** The full record data (optional context). */
+  record?: any;
+  /** Options for the mutation event (e.g., debounce settings). */
+  options?: PushEventOptions;
+  /** Timestamp when the event was created. */
+  createdAt: Date;
+}
+/**
+ * Options for run operations.
+ */
+interface RunOptions {
+  assignedTo?: string;
+  max_retries?: number;
+  retry_strategy?: 'linear' | 'exponential';
+}
+/**
+ * Options for update operations.
+ */
+interface UpdateOptions {
+  /**
+   * Debounce configuration for the update.
+   * If boolean, enables default debounce behavior.
+   */
+  debounced?: boolean | DebounceOptions;
+}
+/**
+ * Configuration options for debouncing updates.
+ */
+interface DebounceOptions {
+  /**
+   * The key to use for debouncing.
+   * - 'recordId': Debounce based on the specific record ID. WARNING: IT WILL ONLY ACCEPT THE LATEST CHANGE AND DOES *NOT* MERGE THE PREVIOUS ONCES. IF YOU ARE UNSURE JUST USE 'recordId_x_fields'.
+   * - 'recordId_x_fields': Debounce based on record ID and specific fields.
+   */
+  key?: 'recordId' | 'recordId_x_fields';
+  /** The debounce delay in milliseconds. */
+  delay?: number;
+}
+//#endregion
+//#region src/modules/auth/events/index.d.ts
+declare const AuthEventTypes: {
+  readonly AuthStateChanged: "AUTH_STATE_CHANGED";
+};
+type AuthEventTypeMap = {
+  [AuthEventTypes.AuthStateChanged]: EventDefinition<typeof AuthEventTypes.AuthStateChanged, string | null>;
+};
+type AuthEventSystem = EventSystem<AuthEventTypeMap>;
+declare function createAuthEventSystem(): AuthEventSystem;
+//#endregion
+//#region src/modules/auth/index.d.ts
+type Prettify<T> = { [K in keyof T]: T[K] } & {};
+type MapColumnType<T extends ColumnSchema> = T['optional'] extends true ? TypeNameToTypeMap[T['type']] | undefined : TypeNameToTypeMap[T['type']];
+type ExtractAccessParams<S extends SchemaStructure, Name extends keyof S['access'], Method extends 'signIn' | 'signup'> = S['access'] extends undefined ? never : S['access'][Name] extends AccessDefinition ? Prettify<{ [K in keyof S['access'][Name][Method]['params']]: MapColumnType<S['access'][Name][Method]['params'][K]> }> : never;
+declare class AuthService<S extends SchemaStructure> {
+  private schema;
+  private remote;
+  private persistenceClient;
+  private logger;
+  token: string | null;
+  currentUser: any | null;
+  isAuthenticated: boolean;
+  isLoading: boolean;
+  private events;
+  get eventSystem(): AuthEventSystem;
+  constructor(schema: S, remote: RemoteDatabaseService, persistenceClient: PersistenceClient, logger: Logger$1);
+  init(): Promise<void>;
+  getAccessDefinition<Name extends keyof S['access']>(name: Name): AccessDefinition | undefined;
+  /**
+   * Subscribe to auth state changes.
+   * callback is called immediately with current value and whenever validation status changes.
+   */
+  subscribe(cb: (userId: string | null) => void): () => void;
+  private notifyListeners;
+  /**
+   * Check for existing session and validate
+   */
+  check(accessToken?: string): Promise<void>;
+  /**
+   * Sign out and clear session
+   */
+  signOut(): Promise<void>;
+  private setSession;
+  signUp<Name extends keyof S['access'] & string>(accessName: Name, params: ExtractAccessParams<S, Name, 'signup'>): Promise<void>;
+  signIn<Name extends keyof S['access'] & string>(accessName: Name, params: ExtractAccessParams<S, Name, 'signIn'>): Promise<void>;
+}
+//#endregion
+//#region src/spooky.d.ts
+declare class SpookyClient<S extends SchemaStructure> {
+  private config;
+  private local;
+  private remote;
+  private persistenceClient;
+  private migrator;
+  private cache;
+  private dataModule;
+  private sync;
+  private devTools;
+  private logger;
+  auth: AuthService<S>;
+  streamProcessor: StreamProcessorService;
+  get remoteClient(): Surreal;
+  get localClient(): Surreal;
+  get pendingMutationCount(): number;
+  subscribeToPendingMutations(cb: (count: number) => void): () => void;
+  constructor(config: SpookyConfig<S>);
+  /**
+   * Setup direct callbacks instead of event subscriptions
+   */
+  private setupCallbacks;
+  init(): Promise<void>;
+  close(): Promise<void>;
+  authenticate(token: string): Promise<surrealdb0.Tokens>;
+  deauthenticate(): Promise<void>;
+  query<Table extends TableNames<S>>(table: Table, options: QueryOptions<TableModel<GetTable<S, Table>>, false>, ttl?: QueryTimeToLive): QueryBuilder<S, Table, SpookyQueryResultPromise>;
+  private initQuery;
+  queryRaw(sql: string, params: Record<string, any>, ttl: QueryTimeToLive): Promise<string>;
+  subscribe(queryHash: string, callback: (records: Record<string, any>[]) => void, options?: {
+    immediate?: boolean;
+  }): Promise<() => void>;
+  run<B extends BackendNames<S>, R extends BackendRoutes<S, B>>(backend: B, path: R, payload: RoutePayload<S, B, R>, options?: RunOptions): Promise<void>;
+  create(id: string, data: Record<string, unknown>): Promise<Record<string, unknown>>;
+  update(table: string, id: string, data: Record<string, unknown>, options?: UpdateOptions): Promise<{
+    [x: string]: /*elided*/any;
+  }>;
+  delete(table: string, id: string): Promise<void>;
+  useRemote<T>(fn: (client: Surreal) => Promise<T> | T): Promise<T>;
+  private persistClientId;
+  private loadOrGenerateClientId;
+}
+//#endregion
+export { AuthEventSystem, AuthEventTypeMap, AuthEventTypes, AuthService, DebounceOptions, EventSubscriptionOptions, type Level, MutationCallback, MutationEvent, MutationEventType, PersistenceClient, QueryConfig, QueryConfigRecord, QueryHash, QueryState, QueryTimeToLive, QueryUpdateCallback, RecordVersionArray, RecordVersionDiff, RunOptions, SpookyClient, SpookyConfig, SpookyQueryResult, SpookyQueryResultPromise, StoreType, UpdateOptions, createAuthEventSystem };