memorix 1.8.9 → 1.9.0

package/src/sync/adapters/github.ts

@@ -0,0 +1,258 @@
+import type { ChangeBatch, SyncCompactReport, SyncPullPage, SyncRemote } from '../types.js';
+import { comparePageKey, decodePageCursor, encodePageCursor } from '../paging.js';
+import { isSafeSyncDeviceId } from '../namespace.js';
+
+interface GitHubRemoteOptions {
+  repo: string;
+  token: string;
+  branch: string;
+  namespace: string;
+  apiBaseUrl: string;
+}
+
+interface GitHubTreeEntry {
+  path?: string;
+  type?: string;
+  sha?: string;
+}
+
+interface GitHubTreeResponse {
+  tree?: GitHubTreeEntry[];
+  truncated?: boolean;
+}
+
+interface GitHubRefResponse {
+  object?: { sha?: string };
+}
+
+interface GitHubCommitResponse {
+  tree?: { sha?: string };
+}
+
+interface GitHubContentResponse {
+  content?: string;
+  encoding?: string;
+}
+
+/**
+ * GitHub is an event relay only. It never receives memorix.db or a WAL file.
+ * Each batch is an immutable JSONL blob under one project/device namespace.
+ */
+export class GitHubRemote implements SyncRemote {
+  readonly kind = 'github';
+  readonly namespace: string;
+  private branchExists = false;
+
+  constructor(private readonly options: GitHubRemoteOptions) {
+    this.namespace = options.namespace;
+  }
+
+  async init(options: { create?: boolean } = {}): Promise<void> {
+    this.branchExists = false;
+    const repository = await this.request<{ default_branch?: string }>(`/repos/${this.options.repo}`);
+    const branch = encodeURIComponent(this.options.branch);
+    try {
+      const ref = await this.request<GitHubRefResponse>(`/repos/${this.options.repo}/git/ref/heads/${branch}`);
+      if (!ref.object?.sha) throw new Error('[memorix] GitHub relay returned a branch without a commit');
+      this.branchExists = true;
+    } catch (error) {
+      if (!isStatus(error, 404) || !repository.default_branch) throw error;
+      if (options.create === false) return;
+      const base = await this.request<{ object?: { sha?: string } }>(
+        `/repos/${this.options.repo}/git/ref/heads/${encodeURIComponent(repository.default_branch)}`,
+      );
+      if (!base.object?.sha) throw new Error('[memorix] GitHub relay could not resolve the default branch');
+      await this.request(`/repos/${this.options.repo}/git/refs`, {
+        method: 'POST',
+        body: JSON.stringify({ ref: `refs/heads/${this.options.branch}`, sha: base.object.sha }),
+      });
+      this.branchExists = true;
+    }
+  }
+
+  async push(batch: ChangeBatch): Promise<void> {
+    assertBatchNamespace(batch, this.namespace);
+    if (!this.branchExists) throw new Error('[memorix] GitHub sync remote is not initialized for writes');
+    const path = `events/${this.namespace}/${encodeURIComponent(batch.deviceId)}/${String(batch.sequence).padStart(20, '0')}.jsonl`;
+    const body = Buffer.from(`${JSON.stringify(batch)}\n`, 'utf8').toString('base64');
+    const apiPath = `/repos/${this.options.repo}/contents/${path.split('/').map(encodeURIComponent).join('/')}`;
+    let lastError: unknown;
+    for (let attempt = 0; attempt < 3; attempt++) {
+      try {
+        await this.request(apiPath, {
+          method: 'PUT',
+          body: JSON.stringify({
+            message: `memorix sync ${this.namespace}/${batch.deviceId}/${batch.sequence}`,
+            content: body,
+            branch: this.options.branch,
+          }),
+        });
+        return;
+      } catch (error) {
+        lastError = error;
+        if (!isStatus(error, 409) && !isStatus(error, 422)) throw error;
+        // A previous process may have committed the immutable event before a
+        // network retry reached us. Treat an identical existing blob as a
+        // successful idempotent write; reject a different payload.
+        const existing = await this.request<GitHubContentResponse>(`${apiPath}?ref=${encodeURIComponent(this.options.branch)}`);
+        if (existing.encoding === 'base64' && normalizeBase64(existing.content) === normalizeBase64(body)) return;
+        throw new Error('[memorix] GitHub relay event path already contains a different payload');
+      }
+    }
+    throw lastError instanceof Error ? lastError : new Error('[memorix] GitHub relay push failed');
+  }
+
+  async pull(since: Record<string, number>, limit: number, pageToken?: string): Promise<SyncPullPage> {
+    assertLimit(limit);
+    if (!this.branchExists) return { batches: [], hasMore: false };
+    const tree = await this.getTree();
+    if (tree.truncated) throw new Error('[memorix] GitHub relay tree is too large; compact the sync repository first');
+
+    const objects = (tree.tree ?? [])
+      .filter((entry): entry is { path: string; type: 'blob'; sha?: string } => typeof entry.path === 'string' && entry.type === 'blob')
+      .map((entry) => {
+        const prefix = `events/${this.namespace}/`;
+        if (!entry.path.startsWith(prefix)) return undefined;
+        const match = new RegExp(`^events/${escapeRegExp(this.namespace)}/([^/]+)/(\\d+)\\.jsonl$`).exec(entry.path);
+        if (!match) return undefined;
+        return { path: entry.path, sha: entry.sha, deviceId: decodeURIComponent(match[1]), sequence: Number(match[2]) };
+      })
+      .filter((entry): entry is { path: string; sha: string | undefined; deviceId: string; sequence: number } => entry !== undefined)
+      .filter((entry) => Number.isSafeInteger(entry.sequence) && entry.sequence > (since[entry.deviceId] ?? 0))
+      .sort((left, right) => left.deviceId === right.deviceId
+        ? left.sequence - right.sequence
+        : left.deviceId < right.deviceId ? -1 : 1);
+
+    const after = decodePageCursor(pageToken);
+    const visibleObjects = objects.filter((object) => !after || comparePageKey(object, after) > 0);
+    const pageObjects = visibleObjects.slice(0, limit);
+    const batches: ChangeBatch[] = [];
+    for (const object of pageObjects) {
+      const path = object.path.split('/').map(encodeURIComponent).join('/');
+      const content = await this.request<GitHubContentResponse>(
+        `/repos/${this.options.repo}/contents/${path}?ref=${encodeURIComponent(this.options.branch)}`,
+      );
+      if (content.encoding !== 'base64' || typeof content.content !== 'string') {
+        throw new Error('[memorix] GitHub relay returned a non-base64 event object');
+      }
+      const text = Buffer.from(content.content.replace(/\s/g, ''), 'base64').toString('utf8');
+      for (const line of text.split(/\r?\n/).filter(Boolean)) batches.push(JSON.parse(line) as ChangeBatch);
+    }
+    const hasMore = visibleObjects.length > pageObjects.length;
+    return {
+      batches,
+      hasMore,
+      nextPageToken: hasMore && pageObjects.length > 0
+        ? encodePageCursor(pageObjects[pageObjects.length - 1])
+        : undefined,
+    };
+  }
+
+  async compact(through: Record<string, number>, options: { dryRun?: boolean } = {}): Promise<SyncCompactReport> {
+    if (!this.branchExists) return { candidates: 0, deleted: 0 };
+    const tree = await this.getTree();
+    const objects = (tree.tree ?? []).flatMap((entry) => {
+      if (!entry.path || !entry.sha) return [];
+      const match = new RegExp(`^events/${escapeRegExp(this.namespace)}/([^/]+)/(\\d+)\\.jsonl$`).exec(entry.path);
+      if (!match) return [];
+      const sequence = Number(match[2]);
+      return sequence <= (through[decodeURIComponent(match[1])] ?? -1) ? [{ path: entry.path, sha: entry.sha }] : [];
+    });
+    if (!options.dryRun) {
+      for (const object of objects) {
+        const path = object.path.split('/').map(encodeURIComponent).join('/');
+        await this.request(`/repos/${this.options.repo}/contents/${path}`, {
+          method: 'DELETE',
+          body: JSON.stringify({ message: `memorix compact ${this.namespace}`, sha: object.sha, branch: this.options.branch }),
+        });
+      }
+    }
+    return { candidates: objects.length, deleted: options.dryRun ? 0 : objects.length };
+  }
+
+  async close(): Promise<void> {}
+
+  private async getTree(): Promise<GitHubTreeResponse> {
+    if (!this.branchExists) return { tree: [] };
+    const ref = await this.request<GitHubRefResponse>(
+      `/repos/${this.options.repo}/git/ref/heads/${encodeURIComponent(this.options.branch)}`,
+    );
+    const commitSha = ref.object?.sha;
+    if (!commitSha) throw new Error('[memorix] GitHub relay returned a branch without a commit');
+    const commit = await this.request<GitHubCommitResponse>(
+      `/repos/${this.options.repo}/git/commits/${encodeURIComponent(commitSha)}`,
+    );
+    if (!commit.tree?.sha) throw new Error('[memorix] GitHub relay could not resolve the branch tree');
+    return this.request<GitHubTreeResponse>(
+      `/repos/${this.options.repo}/git/trees/${encodeURIComponent(commit.tree.sha)}?recursive=1`,
+    );
+  }
+
+  private async request<T>(path: string, init: RequestInit = {}): Promise<T> {
+    const response = await fetch(`${this.options.apiBaseUrl}${path}`, {
+      ...init,
+      headers: {
+        Accept: 'application/vnd.github+json',
+        Authorization: `Bearer ${this.options.token}`,
+        'X-GitHub-Api-Version': '2022-11-28',
+        'Content-Type': 'application/json',
+        ...(init.headers ?? {}),
+      },
+    });
+    if (!response.ok) {
+      const detail = await response.text().catch(() => '');
+      const error = new Error(`[memorix] GitHub relay HTTP ${response.status}${detail ? `: ${detail.slice(0, 240)}` : ''}`);
+      Object.assign(error, { status: response.status });
+      throw error;
+    }
+    return await response.json() as T;
+  }
+}
+
+function assertBatchNamespace(batch: ChangeBatch, namespace: string): void {
+  if (batch.namespace !== namespace) {
+    throw new Error('[memorix] sync batch namespace does not match the GitHub relay');
+  }
+  if (!isSafeSyncDeviceId(batch.deviceId)) {
+    throw new Error('[memorix] sync batch has an invalid device id');
+  }
+}
+
+function assertLimit(limit: number): void {
+  if (!Number.isSafeInteger(limit) || limit < 1) {
+    throw new Error('[memorix] sync pull limit must be a positive integer');
+  }
+}
+
+export function createGitHubRemote(
+  namespace: string,
+  env: NodeJS.ProcessEnv = process.env,
+): GitHubRemote {
+  const repo = required(env, 'MEMORIX_SYNC_GITHUB_REPO');
+  if (!/^[^/]+\/[^/]+$/.test(repo)) throw new Error('MEMORIX_SYNC_GITHUB_REPO must be owner/repository');
+  return new GitHubRemote({
+    namespace,
+    repo,
+    token: required(env, 'MEMORIX_SYNC_GITHUB_TOKEN'),
+    branch: env.MEMORIX_SYNC_GITHUB_BRANCH?.trim() || 'memorix-sync',
+    apiBaseUrl: env.MEMORIX_SYNC_GITHUB_API_URL?.trim() || 'https://api.github.com',
+  });
+}
+
+function required(env: NodeJS.ProcessEnv, name: string): string {
+  const value = env[name]?.trim();
+  if (!value) throw new Error(`[memorix] GitHub sync requires ${name}`);
+  return value;
+}
+
+function isStatus(error: unknown, status: number): boolean {
+  return typeof error === 'object' && error !== null && (error as { status?: unknown }).status === status;
+}
+
+function escapeRegExp(value: string): string {
+  return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+
+function normalizeBase64(value: string | undefined): string {
+  return (value ?? '').replace(/\s/g, '');
+}

package/src/sync/adapters/object-store.ts

@@ -0,0 +1,267 @@
+/**
+ * S3-compatible object-store sync.
+ *
+ * Configuration is read from:
+ * - MEMORIX_SYNC_S3_BUCKET
+ * - MEMORIX_SYNC_S3_ENDPOINT
+ * - MEMORIX_SYNC_S3_ACCESS_KEY_ID
+ * - MEMORIX_SYNC_S3_SECRET_ACCESS_KEY
+ * - MEMORIX_SYNC_S3_REGION
+ * - MEMORIX_SYNC_S3_PREFIX (optional; key namespace so a shared bucket can
+ *   host other data or several independent stores)
+ */
+
+import type { ChangeBatch, SyncCompactReport, SyncPullPage, SyncRemote } from '../types.js';
+import { isSafeSyncDeviceId } from '../namespace.js';
+
+export interface ObjectStoreClient {
+  putIfAbsent(key: string, body: string): Promise<void>;
+  get(key: string): Promise<string | undefined>;
+  put(key: string, body: string): Promise<void>;
+  list(prefix: string, options?: { limit?: number; cursor?: string }): Promise<{ keys: string[]; nextCursor?: string }>;
+  delete(key: string): Promise<void>;
+}
+
+interface BatchObject {
+  key: string;
+  deviceId: string;
+  sequence: number;
+}
+
+const SEQUENCE_WIDTH = 20;
+const BATCH_KEY = /^projects\/([^/]+)\/batches\/([^/]+)\/(\d+)\.jsonl$/;
+
+export class ObjectStoreRemote implements SyncRemote {
+  readonly kind = 's3';
+  readonly namespace: string;
+  private readonly prefix: string;
+
+  constructor(private readonly client: ObjectStoreClient, namespace: string, prefix = '') {
+    this.namespace = namespace;
+    this.prefix = normalizePrefix(prefix);
+  }
+
+  async init(): Promise<void> {}
+
+  async push(batch: ChangeBatch): Promise<void> {
+    assertBatchNamespace(batch, this.namespace);
+    const sequence = String(batch.sequence).padStart(SEQUENCE_WIDTH, '0');
+    await this.client.putIfAbsent(
+      `${this.basePath}batches/${batch.deviceId}/${sequence}.jsonl`,
+      JSON.stringify(batch),
+    );
+  }
+
+  async pull(since: Record<string, number>, limit: number, pageToken?: string): Promise<SyncPullPage> {
+    assertLimit(limit);
+    const objects: BatchObject[] = [];
+    const page = await this.client.list(`${this.basePath}batches/`, { limit, cursor: pageToken });
+    for (const key of page.keys) {
+      const match = BATCH_KEY.exec(key.slice(this.prefix.length));
+      if (match === null) continue;
+      const [, namespace, deviceId, encodedSequence] = match;
+      if (namespace !== this.namespace) continue;
+      const sequence = Number(encodedSequence);
+      if (!Number.isSafeInteger(sequence) || sequence <= (since[deviceId] ?? 0)) continue;
+      objects.push({ key, deviceId, sequence });
+    }
+
+    objects.sort((left, right) => {
+      if (left.deviceId < right.deviceId) return -1;
+      if (left.deviceId > right.deviceId) return 1;
+      return left.sequence - right.sequence;
+    });
+
+    const batches: ChangeBatch[] = [];
+    for (const object of objects.slice(0, limit)) {
+      const body = await this.client.get(object.key);
+      if (body !== undefined) batches.push(JSON.parse(body) as ChangeBatch);
+    }
+    return { batches, hasMore: page.nextCursor !== undefined, nextPageToken: page.nextCursor };
+  }
+
+  async compact(through: Record<string, number>, options: { dryRun?: boolean } = {}): Promise<SyncCompactReport> {
+    let cursor: string | undefined;
+    const keys: string[] = [];
+    do {
+      const page = await this.client.list(`${this.basePath}batches/`, { limit: 1000, cursor });
+      for (const key of page.keys) {
+        const match = BATCH_KEY.exec(key.slice(this.prefix.length));
+        if (!match) continue;
+        const [, namespace, deviceId, encodedSequence] = match;
+        const sequence = Number(encodedSequence);
+        if (namespace === this.namespace && sequence <= (through[deviceId] ?? -1)) keys.push(key);
+      }
+      cursor = page.nextCursor;
+    } while (cursor !== undefined);
+    if (!options.dryRun) for (const key of keys) await this.client.delete(key);
+    return { candidates: keys.length, deleted: options.dryRun ? 0 : keys.length };
+  }
+
+  async close(): Promise<void> {}
+
+  private get basePath(): string {
+    return `${this.prefix}projects/${this.namespace}/`;
+  }
+}
+
+type S3Command = new (input: Record<string, unknown>) => unknown;
+
+interface S3Response {
+  Body?: { transformToString(): Promise<string> };
+  Contents?: Array<{ Key?: string }>;
+  IsTruncated?: boolean;
+  NextContinuationToken?: string;
+  $metadata?: { httpStatusCode?: number };
+}
+
+interface S3Client {
+  send(command: unknown): Promise<S3Response>;
+}
+
+interface S3Module {
+  S3Client: new (config: Record<string, unknown>) => S3Client;
+  GetObjectCommand: S3Command;
+  ListObjectsV2Command: S3Command;
+  PutObjectCommand: S3Command;
+}
+
+export async function createS3ObjectStore(
+  env: NodeJS.ProcessEnv = process.env,
+): Promise<ObjectStoreClient> {
+  const bucket = requiredEnv(env, 'MEMORIX_SYNC_S3_BUCKET');
+  const endpoint = requiredEnv(env, 'MEMORIX_SYNC_S3_ENDPOINT');
+  const accessKeyId = requiredEnv(env, 'MEMORIX_SYNC_S3_ACCESS_KEY_ID');
+  const secretAccessKey = requiredEnv(env, 'MEMORIX_SYNC_S3_SECRET_ACCESS_KEY');
+  const region = requiredEnv(env, 'MEMORIX_SYNC_S3_REGION');
+
+  const moduleName = '@aws-sdk/client-s3';
+  // The SDK is optional by design, so a static import would make every install require it.
+  let sdk: S3Module;
+  try {
+    sdk = await import(moduleName) as unknown as S3Module;
+  } catch (cause) {
+    throw new Error(
+      'S3 sync requires @aws-sdk/client-s3. Install it in the application using Memorix.',
+      { cause },
+    );
+  }
+
+  const s3 = new sdk.S3Client({
+    endpoint,
+    region,
+    credentials: { accessKeyId, secretAccessKey },
+  });
+
+  const client: ObjectStoreClient = {
+    async putIfAbsent(key, body) {
+      try {
+        await s3.send(new sdk.PutObjectCommand({
+          Bucket: bucket,
+          Key: key,
+          Body: body,
+          IfNoneMatch: '*',
+        }));
+      } catch (error) {
+        if (!isPreconditionFailure(error)) throw error;
+        const existing = await client.get(key);
+        if (existing !== body) {
+          throw new Error('[memorix] object-store relay key already contains a different payload');
+        }
+      }
+    },
+
+    async get(key) {
+      try {
+        const response = await s3.send(new sdk.GetObjectCommand({ Bucket: bucket, Key: key }));
+        return response.Body?.transformToString();
+      } catch (error) {
+        if (isNotFound(error)) return undefined;
+        throw error;
+      }
+    },
+
+    async put(key, body) {
+      await s3.send(new sdk.PutObjectCommand({ Bucket: bucket, Key: key, Body: body }));
+    },
+
+    async delete(key) {
+      const DeleteObjectCommand = (sdk as S3Module & { DeleteObjectCommand?: S3Command }).DeleteObjectCommand;
+      if (!DeleteObjectCommand) throw new Error('S3 sync requires DeleteObjectCommand support from @aws-sdk/client-s3');
+      await s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: key }));
+    },
+
+    async list(prefix, options = {}) {
+      const keys: string[] = [];
+      const response = await s3.send(new sdk.ListObjectsV2Command({
+        Bucket: bucket,
+        Prefix: prefix,
+        MaxKeys: Math.max(1, Math.min(1000, options.limit ?? 1000)),
+        ContinuationToken: options.cursor,
+      }));
+      for (const object of response.Contents ?? []) {
+        if (object.Key !== undefined) keys.push(object.Key);
+      }
+      if (response.IsTruncated && !response.NextContinuationToken) {
+        throw new Error('[memorix] S3 sync list was truncated without a continuation token');
+      }
+      return { keys, nextCursor: response.IsTruncated ? response.NextContinuationToken : undefined };
+    },
+  };
+
+  return client;
+}
+
+function requiredEnv(env: NodeJS.ProcessEnv, name: string): string {
+  const value = env[name];
+  if (value === undefined || value.length === 0) {
+    throw new Error(`Missing required environment variable ${name}`);
+  }
+  return value;
+}
+
+function errorDetails(error: unknown): { name?: string; status?: number } {
+  if (typeof error !== 'object' || error === null) return {};
+  const candidate = error as { name?: unknown; $metadata?: { httpStatusCode?: unknown } };
+  return {
+    name: typeof candidate.name === 'string' ? candidate.name : undefined,
+    status: typeof candidate.$metadata?.httpStatusCode === 'number'
+      ? candidate.$metadata.httpStatusCode
+      : undefined,
+  };
+}
+
+function isPreconditionFailure(error: unknown): boolean {
+  const details = errorDetails(error);
+  return details.status === 412 || details.name === 'PreconditionFailed';
+}
+
+function isNotFound(error: unknown): boolean {
+  const details = errorDetails(error);
+  return details.status === 404 || details.name === 'NoSuchKey' || details.name === 'NotFound';
+}
+
+function assertBatchNamespace(batch: ChangeBatch, namespace: string): void {
+  if (batch.namespace !== namespace) {
+    throw new Error('[memorix] sync batch namespace does not match the object-store relay');
+  }
+  if (!isSafeSyncDeviceId(batch.deviceId)) {
+    throw new Error('[memorix] sync batch has an invalid device id');
+  }
+}
+
+function assertLimit(limit: number): void {
+  if (!Number.isSafeInteger(limit) || limit < 1) {
+    throw new Error('[memorix] sync pull limit must be a positive integer');
+  }
+}
+
+function normalizePrefix(value: string): string {
+  const trimmed = value.trim().replace(/^\/+|\/+$/g, '');
+  if (!trimmed) return '';
+  const segments = trimmed.split('/');
+  if (segments.some((segment) => segment.length === 0 || segment === '.' || segment === '..')) {
+    throw new Error('[memorix] MEMORIX_SYNC_S3_PREFIX must be a relative key prefix');
+  }
+  return `${trimmed}/`;
+}