muya 2.1.1 → 2.1.3

package/src/sqlite/create-sqlite.ts

@@ -0,0 +1,273 @@
+/* eslint-disable sonarjs/redundant-type-aliases */
+/* eslint-disable @typescript-eslint/no-shadow */
+/* eslint-disable no-shadow */
+import { createScheduler } from '../scheduler'
+import { shallow } from '../utils/shallow'
+import { selectSql, type CreateState } from './select-sql'
+import { createTable, DEFAULT_STEP_SIZE } from './table/table'
+import type { DbOptions, DocType, Key, MutationResult, SearchOptions, Table } from './table/table.types'
+import type { Where } from './table/where'
+
+type SearchId = string
+const STATE_SCHEDULER = createScheduler()
+
+let stateId = 0
+function getStateId() {
+  return stateId++
+}
+
+export interface SqLiteActions {
+  readonly next: () => Promise<boolean>
+  readonly reset: () => Promise<void>
+}
+export interface SyncTable<Document extends DocType> {
+  // readonly registerSearch: <Selected = Document>(searchId: SearchId, options: SearchOptions<Document, Selected>) => () => void
+  readonly updateSearchOptions: <Selected = Document>(searchId: SearchId, options: SearchOptions<Document, Selected>) => void
+  readonly subscribe: (searchId: SearchId, listener: () => void) => () => void
+  readonly getSnapshot: (searchId: SearchId) => Document[]
+  readonly refresh: (searchId: SearchId) => Promise<void>
+
+  readonly set: (document: Document) => Promise<MutationResult>
+  readonly batchSet: (documents: Document[]) => Promise<MutationResult[]>
+  readonly get: <Selected = Document>(key: Key, selector?: (document: Document) => Selected) => Promise<Selected | undefined>
+
+  readonly delete: (key: Key) => Promise<MutationResult | undefined>
+  readonly search: <Selected = Document>(options?: SearchOptions<Document, Selected>) => AsyncIterableIterator<Selected>
+  readonly count: (options?: { where?: Where<Document> }) => Promise<number>
+  readonly deleteBy: (where: Where<Document>) => Promise<MutationResult[]>
+  readonly destroy: () => void
+  readonly next: (searchId: SearchId) => Promise<boolean>
+
+  readonly select: <Params extends unknown[]>(
+    compute: (...args: Params) => SearchOptions<Document>,
+  ) => CreateState<Document, Params>
+}
+
+interface DataItems<Document extends DocType> {
+  items: Document[]
+  keys: Set<Key>
+  options?: SearchOptions<Document, unknown>
+}
+
+export function createSqliteState<Document extends DocType>(options: DbOptions<Document>): SyncTable<Document> {
+  // const table = await createTable<Document>(options)
+
+  const id = getStateId()
+  function getScheduleId(searchId: SearchId) {
+    return `state-${id}-search-${searchId}`
+  }
+
+  let cachedTable: Table<Document> | undefined
+  async function getTable() {
+    if (!cachedTable) {
+      cachedTable = await createTable<Document>(options)
+    }
+    return cachedTable
+  }
+
+  interface NextResult {
+    document: Document
+    rowId: number
+  }
+  // const emitter = createEmitter<Table<Document>>()
+  const cachedData = new Map<SearchId, DataItems<Document>>()
+  const listeners = new Map<SearchId, () => void>()
+  const iterators = new Map<SearchId, AsyncIterableIterator<NextResult>>()
+
+  async function next(searchId: SearchId, data: DataItems<Document>): Promise<boolean> {
+    const iterator = iterators.get(searchId)
+    const { options = {} } = data
+    const { stepSize = DEFAULT_STEP_SIZE } = options
+    if (!iterator) return false
+    const newItems: Document[] = []
+
+    for (let index = 0; index < stepSize; index++) {
+      const result = await iterator.next()
+      if (result.done) {
+        iterators.delete(searchId)
+        break
+      }
+      newItems.push(result.value.document)
+      data.keys.add(String(result.value.rowId))
+    }
+
+    if (newItems.length === 0) return false
+    if (shallow(data.items, newItems)) return false
+    data.items = [...data.items, ...newItems]
+    return true
+  }
+
+  function notifyListeners(searchId: SearchId) {
+    const searchListeners = listeners.get(searchId)
+    if (searchListeners) {
+      searchListeners()
+    }
+  }
+
+  async function refreshCache(searchId: SearchId) {
+    const table = await getTable()
+    const data = cachedData.get(searchId)
+    if (!data) return
+    const { options } = data
+    const iterator = table.search({ ...options, select: (document, { rowId }) => ({ document, rowId }) })
+    iterators.set(searchId, iterator)
+    data.keys = new Set()
+    data.items = []
+    await next(searchId, data)
+  }
+  async function refresh(searchId: SearchId) {
+    await refreshCache(searchId)
+    notifyListeners(searchId)
+  }
+
+  function handleChange(mutationResult: MutationResult) {
+    const { key, op } = mutationResult
+    // find all cached data with key
+    const searchIds = new Set<SearchId>()
+    for (const [searchId, { keys }] of cachedData) {
+      switch (op) {
+        case 'delete':
+        case 'update': {
+          if (keys.has(String(key))) {
+            searchIds.add(searchId)
+          }
+          break
+        }
+        case 'insert': {
+          // we do not know about the key
+          searchIds.add(searchId)
+          break
+        }
+      }
+    }
+    return searchIds
+  }
+
+  async function handleChanges(mutationResults: MutationResult[]) {
+    const updateSearchIds = new Set<SearchId>()
+    for (const mutationResult of mutationResults) {
+      const searchIds = handleChange(mutationResult)
+      for (const searchId of searchIds) {
+        updateSearchIds.add(searchId)
+      }
+    }
+
+    // const promises = []
+    for (const searchId of updateSearchIds) {
+      const scheduleId = getScheduleId(searchId)
+      STATE_SCHEDULER.schedule(scheduleId, { searchId })
+    }
+  }
+
+  const clearSchedulers = new Set<() => void>()
+
+  function registerData(searchId: SearchId, options?: SearchOptions<Document, unknown>) {
+    if (!cachedData.has(searchId)) {
+      cachedData.set(searchId, { items: [], options, keys: new Set() })
+      if (options) {
+        refresh(searchId)
+      }
+    }
+    const data = cachedData.get(searchId)!
+    if (options) {
+      data.options = options
+    }
+    return data
+  }
+
+  const state: SyncTable<Document> = {
+    async set(document) {
+      const table = await getTable()
+      const changes = await table.set(document)
+      await handleChanges([changes])
+      return changes
+    },
+    async batchSet(documents) {
+      const table = await getTable()
+      const changes = await table.batchSet(documents)
+      await handleChanges(changes)
+      return changes
+    },
+    async delete(key) {
+      const table = await getTable()
+      const changes = await table.delete(key)
+      if (changes) {
+        await handleChanges([changes])
+      }
+      return changes
+    },
+    async deleteBy(where) {
+      const table = await getTable()
+      const changes = await table.deleteBy(where)
+      await handleChanges(changes)
+      return changes
+    },
+    async get(key, selector) {
+      const table = await getTable()
+      return table.get(key, selector)
+    },
+    async *search(options = {}) {
+      const table = await getTable()
+      for await (const item of table.search(options)) {
+        yield item
+      }
+    },
+    async count(options) {
+      const table = await getTable()
+      return await table.count(options)
+    },
+
+    updateSearchOptions(searchId, options) {
+      const data = registerData(searchId, options)
+      data.options = options
+      const scheduleId = getScheduleId(searchId)
+      STATE_SCHEDULER.schedule(scheduleId, { searchId })
+    },
+
+    subscribe(searchId, listener) {
+      const scheduleId = getScheduleId(searchId)
+      const clear = STATE_SCHEDULER.add(scheduleId, {
+        onScheduleDone() {
+          refresh(searchId)
+        },
+      })
+      clearSchedulers.add(clear)
+
+      if (!listeners.has(searchId)) {
+        listeners.set(searchId, listener)
+      }
+      return () => {
+        listeners.delete(searchId)
+        clear()
+        cachedData.delete(searchId)
+      }
+    },
+    getSnapshot(searchId) {
+      const data = registerData(searchId)
+      return data.items
+    },
+    refresh,
+    destroy() {
+      for (const clear of clearSchedulers) clear()
+      cachedData.clear()
+      listeners.clear()
+    },
+    async next(searchId) {
+      const data = cachedData.get(searchId)
+      if (data) {
+        const hasNext = await next(searchId, data)
+        if (hasNext) {
+          notifyListeners(searchId)
+        }
+        return hasNext
+      }
+      return false
+    },
+
+    select(compute) {
+      return selectSql(state, compute)
+    },
+  }
+
+  return state
+}

package/src/sqlite/select-sql.ts

@@ -0,0 +1,55 @@
+import { createState } from '../create-state'
+import type { GetState } from '../types'
+import type { SyncTable } from './create-sqlite'
+import type { DocType } from './table/table.types'
+import type { Where } from './table/where'
+
+export type CreateState<Document, Params extends unknown[]> = (...params: Params) => GetState<Document[]>
+
+export interface SqlSeachOptions<Document extends DocType> {
+  readonly sorBy?: keyof Document
+  readonly order?: 'asc' | 'desc'
+  readonly limit?: number
+  readonly offset?: number
+  readonly where?: Where<Document>
+  readonly stepSize?: number
+}
+
+let stateId = 0
+function getStateId() {
+  stateId++
+  return `${stateId.toString(36)}-sql`
+}
+
+export function selectSql<Document extends DocType, Params extends unknown[] = []>(
+  state: SyncTable<Document>,
+  compute: (...args: Params) => SqlSeachOptions<Document>,
+): CreateState<Document, Params> {
+  const { subscribe, updateSearchOptions } = state
+
+  const result: CreateState<Document, Params> = (...params) => {
+    const searchId = getStateId()
+    const destroy = subscribe(searchId, () => {
+      getState.emitter.emit()
+    })
+
+    const options = compute(...params)
+    const getState = createState<Document[]>({
+      destroy() {
+        destroy()
+        getState.emitter.clear()
+        getState.cache.current = undefined
+      },
+      get() {
+        return state.getSnapshot(searchId)
+      },
+      getSnapshot() {
+        return state.getSnapshot(searchId)
+      },
+    })
+    updateSearchOptions<Document>(searchId, options)
+
+    return getState
+  }
+  return result
+}

package/src/sqlite/table/backend.ts

@@ -0,0 +1,21 @@
+export const IN_MEMORY_DB = ':memory:'
+
+export interface QueryResult {
+  /** The number of rows affected by the query. */
+  rowsAffected: number
+  /**
+   * The last inserted `id`.
+   *
+   * This value is not set for Postgres databases. If the
+   * last inserted id is required on Postgres, the `select` function
+   * must be used, with a `RETURNING` clause
+   * (`INSERT INTO todos (title) VALUES ($1) RETURNING id`).
+   */
+  lastInsertId?: number
+}
+export interface Backend {
+  execute: (query: string, bindValues?: unknown[]) => Promise<QueryResult>
+  select: <T>(query: string, bindValues?: unknown[]) => Promise<T>
+  transaction: (callback: (tx: Backend) => Promise<void>) => Promise<void>
+  path: string
+}

package/src/sqlite/table/bun-backend.ts

@@ -0,0 +1,38 @@
+import { Database, type Statement } from 'bun:sqlite'
+import type { Backend } from './backend'
+import { MapDeque } from './map-deque'
+
+export function bunMemoryBackend(): Backend {
+  const db = Database.open(':memory:')
+  const prepares = new MapDeque<string, Statement>(100)
+  function getStatement(query: string): Statement {
+    if (prepares.has(query)) {
+      return prepares.get(query)!
+    }
+    const stmt = db.prepare(query)
+    prepares.set(query, stmt)
+    return stmt
+  }
+
+  const backend: Backend = {
+    execute: async (query, params = []) => {
+      const q = getStatement(query)
+      const result = q.run(...(params as never[]))
+      return {
+        rowsAffected: result.changes,
+        changes: result.changes,
+      }
+    },
+    transaction: async (callback) => {
+      return db.transaction(() => callback(backend))()
+    },
+    path: db.filename,
+    select: async (query, params = []) => {
+      const q = getStatement(query)
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      const result = q.all(...(params as never[])) as Array<Record<string, any>>
+      return result as never
+    },
+  }
+  return backend
+}

package/src/sqlite/table/map-deque.ts

@@ -0,0 +1,29 @@
+export class MapDeque<K, V> extends Map<K, V> {
+  constructor(
+    private maxSize: number,
+    entries?: ReadonlyArray<readonly [K, V]> | null,
+  ) {
+    super(entries)
+    if (this.maxSize <= 0) {
+      throw new RangeError('maxSize must be greater than 0')
+    }
+  }
+
+  override set(key: K, value: V): this {
+    if (this.has(key)) {
+      super.set(key, value)
+      return this
+    }
+
+    if (this.size >= this.maxSize) {
+      const firstKey = this.keys().next().value
+      if (firstKey !== undefined) {
+        this.delete(firstKey)
+      }
+    }
+
+    super.set(key, value)
+
+    return this
+  }
+}

package/src/sqlite/table/table.ts

@@ -0,0 +1,200 @@
+// table.ts
+/* eslint-disable sonarjs/different-types-comparison */
+/* eslint-disable sonarjs/cognitive-complexity */
+/* eslint-disable @typescript-eslint/no-shadow */
+/* eslint-disable no-shadow */
+import type { Table, DbOptions, DocType, Key, SearchOptions, MutationResult } from './table.types'
+import { getWhereQuery, type Where } from './where'
+
+const DELETE_IN_CHUNK = 500 // keep well below SQLite's default 999 parameter limit
+export const DEFAULT_STEP_SIZE = 100
+export async function createTable<Document extends DocType>(options: DbOptions<Document>): Promise<Table<Document>> {
+  const { backend, tableName, indexes, key } = options
+  const hasUserKey = key !== undefined
+
+  // Schema
+  if (hasUserKey) {
+    await backend.execute(`
+      CREATE TABLE IF NOT EXISTS ${tableName} (
+        key TEXT PRIMARY KEY,
+        data TEXT NOT NULL
+      );
+    `)
+  } else {
+    await backend.execute(`
+      CREATE TABLE IF NOT EXISTS ${tableName} (
+        data TEXT NOT NULL
+      );
+    `)
+  }
+
+  // JSON expression indexes for fields under data
+  for (const index of indexes ?? []) {
+    const idx = String(index)
+    await backend.execute(`CREATE INDEX IF NOT EXISTS idx_${tableName}_${idx} ON ${tableName} (json_extract(data, '$.${idx}'));`)
+  }
+
+  function getKeyFromDocument(document: Document): Key | undefined {
+    return hasUserKey ? (document[key as keyof Document] as unknown as Key | undefined) : undefined
+  }
+
+  async function getChanges(conn: typeof backend): Promise<number> {
+    const r = await conn.select<Array<{ c: number }>>(`SELECT changes() AS c`)
+    return r[0]?.c ?? 0
+  }
+
+  const table: Table<Document> = {
+    backend,
+
+    async set(document, backendOverride) {
+      const db = backendOverride ?? backend
+      const json = JSON.stringify(document)
+
+      if (hasUserKey) {
+        const id = getKeyFromDocument(document)
+        if (id === undefined || id === null) {
+          throw new Error(
+            `Document is missing the configured key "${String(key)}". Provide it or create the table without "key".`,
+          )
+        }
+
+        // Fast path: UPDATE first
+        await db.execute(`UPDATE ${tableName} SET data = ? WHERE key = ?`, [json, id])
+        const updated = await getChanges(db)
+        if (updated === 1) return { key: id, op: 'update' }
+
+        // No row updated => try INSERT
+        try {
+          await db.execute(`INSERT INTO ${tableName} (key, data) VALUES (?, ?)`, [id, json])
+          return { key: id, op: 'insert' }
+        } catch {
+          await db.execute(`UPDATE ${tableName} SET data = ? WHERE key = ?`, [json, id])
+          return { key: id, op: 'update' }
+        }
+      }
+
+      // ROWID mode
+      await db.execute(`INSERT INTO ${tableName} (data) VALUES (?)`, [json])
+      const rows = await db.select<Array<{ id: number }>>(`SELECT last_insert_rowid() AS id`)
+      const rowid = rows[0]?.id
+      if (typeof rowid !== 'number') throw new Error('Failed to retrieve last_insert_rowid()')
+      const result: MutationResult = { key: rowid, op: 'insert' }
+      return result
+    },
+
+    // --- FIXED: include rowid ---
+    async get<Selected = Document>(
+      keyValue: Key,
+      selector: (document: Document, meta: { rowid: number }) => Selected = (d, _m) => d as unknown as Selected,
+    ) {
+      const whereKey = hasUserKey ? `key = ?` : `rowid = ?`
+      const result = await backend.select<Array<{ data: string; rowid: number }>>(
+        `SELECT rowid, data FROM ${tableName} WHERE ${whereKey}`,
+        [keyValue],
+      )
+      if (result.length === 0) return
+      const [item] = result
+      const { data, rowid } = item
+      const document = JSON.parse(data) as Document
+      return selector(document, { rowid }) as Selected
+    },
+
+    async delete(keyValue: Key) {
+      const whereKey = hasUserKey ? `key = ?` : `rowid = ?`
+      await backend.execute(`DELETE FROM ${tableName} WHERE ${whereKey}`, [keyValue])
+      const changed = await backend.select<Array<{ c: number }>>(`SELECT changes() AS c`)
+      if ((changed[0]?.c ?? 0) > 0) {
+        return { key: keyValue, op: 'delete' }
+      }
+      return
+    },
+
+    // --- FIXED: include rowid in search ---
+    async *search<Selected = Document>(options: SearchOptions<Document, Selected> = {}): AsyncIterableIterator<Selected> {
+      const {
+        sorBy,
+        order = 'asc',
+        limit,
+        offset = 0,
+        where,
+        select = (document, _meta) => document as unknown as Selected,
+        stepSize = DEFAULT_STEP_SIZE,
+      } = options
+
+      let baseQuery = `SELECT rowid, data FROM ${tableName}`
+      if (where) baseQuery += ' ' + getWhereQuery(where)
+
+      let yielded = 0
+      let currentOffset = offset
+      while (true) {
+        let query = baseQuery
+
+        if (sorBy) {
+          query += ` ORDER BY json_extract(data, '$.${String(sorBy)}') COLLATE NOCASE ${order.toUpperCase()}`
+        } else {
+          query += hasUserKey ? ` ORDER BY key COLLATE NOCASE ${order.toUpperCase()}` : ` ORDER BY rowid ${order.toUpperCase()}`
+        }
+
+        const batchLimit = limit ? Math.min(stepSize, limit - yielded) : stepSize
+        query += ` LIMIT ${batchLimit} OFFSET ${currentOffset}`
+
+        const results = await backend.select<Array<{ rowid: number; data: string }>>(query)
+        if (results.length === 0) break
+
+        for (const { rowid, data } of results) {
+          if (limit && yielded >= limit) return
+          const document = JSON.parse(data) as Document
+          yield select(document, { rowId: rowid }) as Selected
+          yielded++
+        }
+
+        if (results.length < batchLimit || (limit && yielded >= limit)) break
+        currentOffset += results.length
+      }
+    },
+
+    async count(options: { where?: Where<Document> } = {}) {
+      const { where } = options
+      let query = `SELECT COUNT(*) as count FROM ${tableName}`
+      if (where) query += ' ' + getWhereQuery(where)
+      const result = await backend.select<Array<{ count: number }>>(query)
+      return result[0]?.count ?? 0
+    },
+
+    async deleteBy(where: Where<Document>) {
+      const whereQuery = getWhereQuery(where)
+      const keyCol = hasUserKey ? 'key' : 'rowid'
+
+      const results: MutationResult[] = []
+      await backend.transaction(async (tx) => {
+        const rows = await tx.select<Array<{ k: Key }>>(`SELECT ${keyCol} AS k, rowid FROM ${tableName} ${whereQuery}`)
+        if (rows.length === 0) return
+
+        const allKeys = rows.map((r) => r.k)
+
+        for (let index = 0; index < allKeys.length; index += DELETE_IN_CHUNK) {
+          const chunk = allKeys.slice(index, index + DELETE_IN_CHUNK)
+          const placeholders = chunk.map(() => '?').join(',')
+          await tx.execute(`DELETE FROM ${tableName} WHERE ${keyCol} IN (${placeholders})`, chunk as unknown as unknown[])
+        }
+
+        for (const k of allKeys) results.push({ key: k, op: 'delete' })
+      })
+
+      return results
+    },
+
+    async batchSet(documents: Document[]) {
+      const mutations: MutationResult[] = []
+      await backend.transaction(async (tx) => {
+        for (const document of documents) {
+          const m = await table.set(document, tx)
+          mutations.push(m)
+        }
+      })
+      return mutations
+    },
+  }
+
+  return table
+}

package/src/sqlite/table/table.types.ts

@@ -0,0 +1,55 @@
+// table.types.ts
+import type { Backend } from './backend'
+import type { Where } from './where'
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export type DocType = { [key: string]: any }
+export type KeyTypeAvailable = 'string' | 'number'
+
+export interface DbOptions<Document extends DocType> {
+  readonly sorBy?: keyof Document
+  readonly order?: 'asc' | 'desc'
+  readonly tableName: string
+  readonly indexes?: Array<keyof Document>
+  readonly backend: Backend
+  /**
+   * Optional key. If omitted, the table uses implicit SQLite ROWID as the key.
+   */
+  readonly key?: keyof Document
+}
+
+interface DbNotGeneric {
+  readonly backend: Backend
+}
+
+export interface SearchOptions<Document extends DocType, Selected = Document> {
+  readonly sorBy?: keyof Document
+  readonly order?: 'asc' | 'desc'
+  readonly limit?: number
+  readonly offset?: number
+  readonly where?: Where<Document>
+  readonly stepSize?: number
+  /**
+   * Naive projection. Prefer specialized queries for heavy fan-out graphs.
+   */
+  readonly select?: (document: Document, meta: { rowId: number }) => Selected
+}
+
+export type Key = string | number
+
+export type MutationOp = 'insert' | 'update' | 'delete'
+export interface MutationResult {
+  key: Key
+  op: MutationOp
+}
+
+export interface Table<Document extends DocType> extends DbNotGeneric {
+  readonly set: (document: Document, backendOverride?: Backend) => Promise<MutationResult>
+  readonly batchSet: (documents: Document[]) => Promise<MutationResult[]>
+  readonly get: <Selected = Document>(key: Key, selector?: (document: Document) => Selected) => Promise<Selected | undefined>
+
+  readonly delete: (key: Key) => Promise<MutationResult | undefined>
+  readonly search: <Selected = Document>(options?: SearchOptions<Document, Selected>) => AsyncIterableIterator<Selected>
+  readonly count: (options?: { where?: Where<Document> }) => Promise<number>
+  readonly deleteBy: (where: Where<Document>) => Promise<MutationResult[]>
+}