@dxos/client-services 0.8.4-main.fffef41 → 0.9.0

package/src/packlets/services/sqlite-storage.ts

@@ -0,0 +1,390 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as SqlClient from '@effect/sql/SqlClient';
+import type * as SqlError from '@effect/sql/SqlError';
+import * as Effect from 'effect/Effect';
+import type { Callback, FileStat, RandomAccessStorage } from 'random-access-storage';
+
+import { RuntimeProvider } from '@dxos/effect';
+import { log } from '@dxos/log';
+import { Directory, type File, StorageType, type Storage, wrapFile } from '@dxos/random-access-storage';
+import { SqlTransaction } from '@dxos/sql-sqlite';
+
+// SqlTransaction.SqlTransaction is the Tag class exported from the SqlTransaction namespace.
+type SqlTransactionTag = SqlTransaction.SqlTransaction;
+
+export type SqliteStorageOptions = {
+  runtime: RuntimeProvider.RuntimeProvider<SqlClient.SqlClient | SqlTransactionTag>;
+};
+
+/** Minimal cross-platform EventEmitter needed by the RandomAccessStorage contract. */
+class BaseEventEmitter {
+  readonly #events: Record<string, ((...args: unknown[]) => void)[]> = Object.create(null);
+
+  on(event: string, listener: (...args: unknown[]) => void): this {
+    (this.#events[event] ??= []).push(listener);
+    return this;
+  }
+  off(event: string, listener: (...args: unknown[]) => void): this {
+    const list = this.#events[event];
+    if (list) {
+      const idx = list.indexOf(listener);
+      if (idx !== -1) {
+        list.splice(idx, 1);
+      }
+    }
+    return this;
+  }
+  emit(event: string, ...args: unknown[]): boolean {
+    const list = this.#events[event];
+    if (!list?.length) {
+      return false;
+    }
+    for (const fn of [...list]) {
+      fn(...args);
+    }
+    return true;
+  }
+  addListener = this.on;
+  removeListener = this.off;
+  once(event: string, listener: (...args: unknown[]) => void): this {
+    const wrapper = (...args: unknown[]) => {
+      this.off(event, wrapper);
+      listener(...args);
+    };
+    return this.on(event, wrapper);
+  }
+  removeAllListeners(event?: string): this {
+    if (event) {
+      delete this.#events[event];
+    } else {
+      for (const key of Object.keys(this.#events)) {
+        delete this.#events[key];
+      }
+    }
+    return this;
+  }
+  listeners(event: string): ((...args: unknown[]) => void)[] {
+    return this.#events[event] ?? [];
+  }
+  rawListeners(event: string): ((...args: unknown[]) => void)[] {
+    return this.#events[event] ?? [];
+  }
+  listenerCount(event: string): number {
+    return this.#events[event]?.length ?? 0;
+  }
+  prependListener(event: string, listener: (...args: unknown[]) => void): this {
+    (this.#events[event] ??= []).unshift(listener);
+    return this;
+  }
+  prependOnceListener(event: string, listener: (...args: unknown[]) => void): this {
+    const wrapper = (...args: unknown[]) => {
+      this.off(event, wrapper);
+      listener(...args);
+    };
+    return this.prependListener(event, wrapper);
+  }
+  eventNames(): string[] {
+    return Object.keys(this.#events);
+  }
+  getMaxListeners(): number {
+    return 0;
+  }
+  setMaxListeners(): this {
+    return this;
+  }
+}
+
+/**
+ * SQLite-backed random-access file for hypercore storage.
+ * Stores each file as a blob in `hypercore_files`. Read-modify-write for writes.
+ */
+class SqliteRandomAccessFile extends BaseEventEmitter implements RandomAccessStorage {
+  readonly opened: boolean = false;
+  readonly suspended: boolean = false;
+  readonly writing: boolean = false;
+  readonly readable: boolean = true;
+  readonly writable: boolean = true;
+  readonly deletable: boolean = true;
+  readonly truncatable: boolean = false;
+  // Setting statable=false prevents hypercore's statAndReadAll() from computing a negative
+  // read-size (stat.size - offset) for new empty files, which causes an infinite async loop.
+  readonly statable: boolean = false;
+
+  #closed = false;
+  #unlinked = false;
+  #destroyed = false;
+
+  get closed(): boolean {
+    return this.#closed;
+  }
+
+  get unlinked(): boolean {
+    return this.#unlinked;
+  }
+
+  get destroyed(): boolean {
+    return this.#destroyed;
+  }
+
+  // Filesystem-like metadata for compatibility with wrapFile.
+  readonly directory: string;
+  readonly filename: string;
+
+  #buffer: Buffer = Buffer.alloc(0);
+  #loaded = false;
+  #loading: Promise<void> | null = null;
+
+  constructor(
+    private readonly filePath: string,
+    private readonly runtime: RuntimeProvider.RuntimeProvider<SqlClient.SqlClient | SqlTransactionTag>,
+  ) {
+    super();
+    const parts = filePath.split('/');
+    this.filename = parts.pop() ?? '';
+    this.directory = parts.join('/');
+  }
+
+  private _ensureLoaded(): Promise<void> {
+    if (this.#loaded) {
+      return Promise.resolve();
+    }
+    // If the file was closed (e.g. the storage was torn down during shutdown) before its initial
+    // load completed, don't start the DB read. The underlying SQL connection may already be
+    // closing, and the read would reject with "database connection is not open" as an unhandled
+    // rejection — a background read racing teardown. #buffer already defaults to empty, so reads
+    // resolve against an empty file.
+    if (this.#closed) {
+      return Promise.resolve();
+    }
+    if (!this.#loading) {
+      this.#loading = this._loadFromDb();
+    }
+    return this.#loading;
+  }
+
+  private async _loadFromDb(): Promise<void> {
+    const filePath = this.filePath;
+    const rows = await RuntimeProvider.runPromise(this.runtime)(
+      Effect.gen(function* () {
+        const sql = yield* SqlClient.SqlClient;
+        return yield* sql<{ data: Uint8Array }>`SELECT data FROM hypercore_files WHERE path = ${filePath}`;
+      }),
+    );
+    this.#buffer = rows.length > 0 ? Buffer.from(rows[0].data) : Buffer.alloc(0);
+    this.#loaded = true;
+  }
+
+  private async _saveToDb(): Promise<void> {
+    const filePath = this.filePath;
+    const data = this.#buffer;
+    await RuntimeProvider.runPromise(this.runtime)(
+      Effect.gen(function* () {
+        const sql = yield* SqlClient.SqlClient;
+        yield* sql`INSERT OR REPLACE INTO hypercore_files (path, data) VALUES (${filePath}, ${data})`;
+      }),
+    );
+  }
+
+  write(offset: number, data: Buffer, cb: Callback<any>): void {
+    this._ensureLoaded()
+      .then(() => {
+        const end = offset + data.length;
+        if (end > this.#buffer.length) {
+          const enlarged = Buffer.alloc(end);
+          this.#buffer.copy(enlarged);
+          this.#buffer = enlarged;
+        }
+        data.copy(this.#buffer, offset);
+        return this._saveToDb();
+      })
+      .then(() => cb(null))
+      .catch((err) => cb(err));
+  }
+
+  read(offset: number, size: number, cb: Callback<Buffer>): void {
+    this._ensureLoaded()
+      .then(() => {
+        if (size === 0) {
+          cb(null, Buffer.alloc(0));
+          return;
+        }
+        if (offset + size > this.#buffer.length) {
+          cb(new Error(`Could not satisfy length ${size} at offset ${offset} (file size: ${this.#buffer.length})`));
+          return;
+        }
+        cb(null, Buffer.from(this.#buffer.subarray(offset, offset + size)));
+      })
+      .catch((err) => cb(err));
+  }
+
+  del(offset: number, size: number, cb: Callback<any>): void {
+    this._ensureLoaded()
+      .then(() => {
+        const end = Math.min(offset + size, this.#buffer.length);
+        if (offset < end) {
+          this.#buffer.fill(0, offset, end);
+        }
+        return this._saveToDb();
+      })
+      .then(() => cb(null))
+      .catch((err) => cb(err));
+  }
+
+  stat(cb: Callback<FileStat>): void {
+    this._ensureLoaded()
+      .then(() => {
+        cb(null, { size: this.#buffer.length });
+      })
+      .catch((err) => cb(err));
+  }
+
+  close(cb: Callback<Error>): void {
+    this.#closed = true;
+    // Drain any in-flight initial load before reporting closed, so a read never races a
+    // torn-down SQL connection (avoids "database connection is not open" unhandled rejections).
+    (this.#loading ?? Promise.resolve()).then(
+      () => cb(null),
+      () => cb(null),
+    );
+  }
+
+  destroy(cb: Callback<Error>): void {
+    const filePath = this.filePath;
+    RuntimeProvider.runPromise(this.runtime)(
+      Effect.gen(function* () {
+        const sql = yield* SqlClient.SqlClient;
+        yield* sql`DELETE FROM hypercore_files WHERE path = ${filePath}`;
+      }),
+    )
+      .then(() => {
+        this.#buffer = Buffer.alloc(0);
+        this.#destroyed = true;
+        this.#unlinked = true;
+        this.#closed = true;
+        cb(null);
+      })
+      .catch((err) => cb(err));
+  }
+}
+
+/**
+ * SQLite-backed Storage implementation for hypercore feeds.
+ * Table: hypercore_files(path TEXT PRIMARY KEY, data BLOB NOT NULL DEFAULT x'').
+ */
+export class SqliteStorage implements Storage {
+  readonly path: string;
+  readonly type = StorageType.NODE;
+
+  readonly #runtime: RuntimeProvider.RuntimeProvider<SqlClient.SqlClient | SqlTransactionTag>;
+  readonly #files = new Map<string, File>();
+  readonly #nativeFiles = new Map<string, SqliteRandomAccessFile>();
+  #closed = false;
+
+  constructor({ runtime }: SqliteStorageOptions, path = '/sqlite-feeds') {
+    this.#runtime = runtime;
+    this.path = path;
+  }
+
+  readonly migrate: Effect.Effect<void, SqlError.SqlError, SqlClient.SqlClient | SqlTransactionTag> = Effect.fn(
+    'SqliteStorage.migrate',
+  )(() =>
+    Effect.gen(function* () {
+      const sql = yield* SqlClient.SqlClient;
+      yield* sql`CREATE TABLE IF NOT EXISTS hypercore_files (
+          path TEXT PRIMARY KEY,
+          data BLOB NOT NULL DEFAULT x''
+        )`;
+      log('hypercore_files table ready');
+    }).pipe(Effect.withSpan('SqliteStorage.migrate')),
+  )();
+
+  get size(): number {
+    return this.#files.size;
+  }
+
+  createDirectory(sub = ''): Directory {
+    const dirPath = sub ? `${this.path}/${sub}` : this.path;
+    const runtime = this.#runtime;
+    const files = this.#files;
+    const nativeFiles = this.#nativeFiles;
+
+    const getOrCreateFile = (path: string, filename: string): File => {
+      const fullPath = `${path}/${filename}`;
+      const existingNative = nativeFiles.get(fullPath);
+      if (existingNative && !existingNative.destroyed) {
+        return files.get(fullPath)!;
+      }
+      const native = new SqliteRandomAccessFile(fullPath, runtime);
+      const file = wrapFile(native, StorageType.NODE);
+      files.set(fullPath, file);
+      nativeFiles.set(fullPath, native);
+      return file;
+    };
+
+    const list = async (path: string): Promise<string[]> => {
+      const prefix = path.endsWith('/') ? path : path + '/';
+      const rows = await RuntimeProvider.runPromise(runtime)(
+        Effect.gen(function* () {
+          const sql = yield* SqlClient.SqlClient;
+          return yield* sql<{
+            path: string;
+          }>`SELECT path FROM hypercore_files WHERE path = ${path} OR path LIKE ${prefix + '%'}`;
+        }),
+      );
+      return rows.map((row) => row.path.replace(path + '/', '').replace(path, ''));
+    };
+
+    const remove = async (): Promise<void> => {
+      const prefix = dirPath.endsWith('/') ? dirPath : dirPath + '/';
+      await RuntimeProvider.runPromise(runtime)(
+        Effect.gen(function* () {
+          const sql = yield* SqlClient.SqlClient;
+          yield* sql`DELETE FROM hypercore_files WHERE path = ${dirPath} OR path LIKE ${prefix + '%'}`;
+        }),
+      );
+      // Clean up in-memory file cache.
+      for (const [filePath] of files) {
+        if (filePath === dirPath || filePath.startsWith(prefix)) {
+          files.delete(filePath);
+        }
+      }
+    };
+
+    return new Directory({
+      type: StorageType.NODE,
+      path: dirPath,
+      list,
+      getOrCreateFile,
+      remove,
+    });
+  }
+
+  async reset(): Promise<void> {
+    log('SqliteStorage: clearing all hypercore files');
+    await RuntimeProvider.runPromise(this.#runtime)(
+      Effect.gen(function* () {
+        const sql = yield* SqlClient.SqlClient;
+        yield* sql`DELETE FROM hypercore_files`;
+      }),
+    );
+    this.#files.clear();
+  }
+
+  async close(): Promise<void> {
+    if (this.#closed) {
+      return;
+    }
+    this.#closed = true;
+    // Close each native file before the SQL connection is torn down. `close()` marks the file
+    // closed (so any subsequent read skips the DB) and drains its in-flight initial load —
+    // preventing "database connection is not open" unhandled rejections from background reads
+    // that race shutdown (e.g. a test client closing while a background sync read is in flight).
+    const natives = [...this.#nativeFiles.values()];
+    await Promise.all(natives.map((native) => new Promise<void>((resolve) => native.close(() => resolve()))));
+    this.#files.clear();
+    this.#nativeFiles.clear();
+  }
+}

package/src/packlets/space-export/archive-format.ts

@@ -0,0 +1,42 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { SpaceArchive } from '@dxos/protocols/proto/dxos/client/services';
+
+const JSON_EXTENSIONS = ['.dx.json', '.dx.json.gz', '.json'];
+const BINARY_EXTENSIONS = ['.tar', '.tar.gz'];
+
+/**
+ * Detect the format of a space archive.
+ *
+ * Uses filename extension as the primary signal, and falls back to sniffing
+ * the first bytes of the archive contents to distinguish JSON from tar.
+ */
+export const detectSpaceArchiveFormat = (archive: Pick<SpaceArchive, 'filename' | 'contents'>): SpaceArchive.Format => {
+  const filename = archive.filename?.toLowerCase() ?? '';
+  if (JSON_EXTENSIONS.some((ext) => filename.endsWith(ext))) {
+    return SpaceArchive.Format.JSON;
+  }
+  if (BINARY_EXTENSIONS.some((ext) => filename.endsWith(ext))) {
+    return SpaceArchive.Format.BINARY;
+  }
+
+  // Fall back to byte sniffing: JSON archives start with '{' (0x7B) or whitespace.
+  const bytes = archive.contents;
+  if (bytes && bytes.length > 0) {
+    for (let i = 0; i < Math.min(bytes.length, 16); i++) {
+      const byte = bytes[i];
+      // Skip whitespace.
+      if (byte === 0x20 || byte === 0x09 || byte === 0x0a || byte === 0x0d) {
+        continue;
+      }
+      if (byte === 0x7b /* '{' */) {
+        return SpaceArchive.Format.JSON;
+      }
+      break;
+    }
+  }
+
+  return SpaceArchive.Format.BINARY;
+};

package/src/packlets/space-export/index.ts

@@ -2,5 +2,8 @@
 // Copyright 2025 DXOS.org
 //
 
-export * from './space-archive-writer';
+export * from './archive-format';
+export * from './serialized-space-reader';
+export * from './serialized-space-writer';
 export * from './space-archive-reader';
+export * from './space-archive-writer';

package/src/packlets/space-export/serialized-space-reader.ts

@@ -0,0 +1,129 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { type Obj, Type } from '@dxos/echo';
+import { type SerializedSpace } from '@dxos/echo-client';
+import { type DatabaseDirectory, type EntityStructure } from '@dxos/echo-protocol';
+import { assertArgument } from '@dxos/invariant';
+import { URI } from '@dxos/keys';
+import { type SpaceArchive } from '@dxos/protocols/proto/dxos/client/services';
+
+const ATTR_TYPE = '@type';
+const ATTR_META = '@meta';
+const ATTR_DELETED = '@deleted';
+const ATTR_PARENT = '@parent';
+const ATTR_RELATION_SOURCE = '@relationSource';
+const ATTR_RELATION_TARGET = '@relationTarget';
+
+/**
+ * Type URI of the meta-schema (`TypeSchema` / `Type.Type`). Objects with this
+ * `@type` represent persisted ECHO type definitions and must be branded
+ * `system.kind = 'type'` so `Filter.type(Type.Type)` and `Type.isType` recognize
+ * them after import. Anything else defaults to `'object'` (or `'relation'` when
+ * source/target attrs are present).
+ */
+const TYPE_KIND_SCHEMA_URI = Type.getURI(Type.Type).toString();
+
+const INTERNAL_KEYS: ReadonlySet<string> = new Set([
+  'id',
+  ATTR_TYPE,
+  ATTR_META,
+  ATTR_DELETED,
+  ATTR_PARENT,
+  ATTR_RELATION_SOURCE,
+  ATTR_RELATION_TARGET,
+]);
+
+/**
+ * Parse a JSON space archive.
+ *
+ * The archive contents are expected to be the UTF-8 encoding of
+ * `JSON.stringify(serializedSpace)`.
+ */
+export const readSerializedSpaceArchive = (archive: SpaceArchive): SerializedSpace => {
+  const text = new TextDecoder().decode(archive.contents);
+  const parsed = JSON.parse(text) as SerializedSpace;
+  assertArgument(typeof parsed === 'object' && parsed !== null, 'archive', 'Invalid JSON archive payload');
+  assertArgument(typeof parsed.version === 'number', 'archive', 'Missing SerializedSpace.version');
+  assertArgument(Array.isArray(parsed.objects), 'archive', 'Missing SerializedSpace.objects');
+  return parsed;
+};
+
+/**
+ * Convert an {@link Obj.JSON} back into an internal {@link EntityStructure} suitable
+ * for embedding into a {@link DatabaseDirectory}.
+ */
+export const objJsonToObjectStructure = (obj: Obj.JSON): EntityStructure => {
+  const data: Record<string, unknown> = {};
+  for (const [key, value] of Object.entries(obj)) {
+    if (INTERNAL_KEYS.has(key)) {
+      continue;
+    }
+    data[key] = value;
+  }
+
+  const system: NonNullable<EntityStructure['system']> = {};
+
+  const type = obj[ATTR_TYPE];
+  if (type) {
+    system.type = { '/': URI.make(type) };
+  }
+
+  const parent = (obj as any)[ATTR_PARENT];
+  if (typeof parent === 'string') {
+    system.parent = { '/': URI.make(parent) };
+  }
+
+  const relationSource = (obj as any)[ATTR_RELATION_SOURCE];
+  const relationTarget = (obj as any)[ATTR_RELATION_TARGET];
+  if (typeof relationSource === 'string' || typeof relationTarget === 'string') {
+    system.kind = 'relation';
+    if (typeof relationSource === 'string') {
+      system.source = { '/': URI.make(relationSource) };
+    }
+    if (typeof relationTarget === 'string') {
+      system.target = { '/': URI.make(relationTarget) };
+    }
+    // TODO(wittjosiah): This is fragile, will break if the type URI changes.
+  } else if (type === TYPE_KIND_SCHEMA_URI) {
+    system.kind = 'type';
+  } else {
+    system.kind = 'object';
+  }
+
+  if ((obj as any)[ATTR_DELETED]) {
+    system.deleted = true;
+  }
+
+  const meta = (obj as any)[ATTR_META];
+  return {
+    system,
+    meta: {
+      keys: meta?.keys ?? [],
+      ...(meta?.tags ? { tags: meta.tags } : {}),
+      // Preserve registry-provenance fields so persisted `Type.Type` entities
+      // round-trip with their typename / semver (see the symmetric write in
+      // `objectStructureToObjJson`).
+      ...(meta?.key !== undefined ? { key: meta.key } : {}),
+      ...(meta?.version !== undefined ? { version: meta.version } : {}),
+    },
+    data,
+  };
+};
+
+/**
+ * Build a new {@link DatabaseDirectory} containing every object from the archive,
+ * keyed by object id. The caller is responsible for stamping the `access.spaceKey`
+ * and version fields after the document is created.
+ */
+export const buildDatabaseDirectoryFromObjects = (objects: readonly Obj.JSON[]): DatabaseDirectory => {
+  const map: Record<string, EntityStructure> = {};
+  for (const obj of objects) {
+    map[obj.id] = objJsonToObjectStructure(obj);
+  }
+  return {
+    objects: map,
+    links: {},
+  };
+};