@pylonsync/sync 0.22.47 → 0.22.58

package/dist/upload-queue.d.ts

@@ -0,0 +1,190 @@
+import type { OutboxItem } from "./mutation-queue";
+import type { TransportConfig } from "./transport";
+import { type FileVisibility, type UploadedFile } from "./upload";
+/** Where an upload is attached when it finishes: `entity`/`id` gets
+ *  `field` = the file id, and `urlField` (when set) = the file URL. */
+export interface UploadAttachTarget {
+    entity: string;
+    id: string;
+    field: string;
+    urlField?: string;
+}
+/** One queued upload, as stored. */
+export interface QueuedUpload {
+    /** `upl_...`. The record's field holds `local:<localId>` until the
+     *  upload finishes. */
+    localId: string;
+    blob: Blob;
+    filename: string;
+    contentType: string;
+    size: number;
+    visibility: FileVisibility;
+    /** User id the upload was made as (`null` = anonymous). */
+    owner: string | null;
+    /** Made before this run knew who is signed in; stamped by the first
+     *  resolved session. */
+    ownerPending?: boolean;
+    attach: UploadAttachTarget;
+    createdAt: number;
+    status: "pending" | "failed";
+    error?: string;
+    errorCode?: string;
+    /** Set once the server confirmed the file, before the attach update is
+     *  queued. A restart in between attaches without uploading again. */
+    uploaded?: UploadedFile;
+}
+/** Persistence for queued uploads. */
+export interface UploadStore {
+    put(record: QueuedUpload): Promise<void>;
+    delete(localId: string): Promise<void>;
+    list(): Promise<QueuedUpload[]>;
+}
+/** The prefix of a field value that names a queued upload. */
+export declare const LOCAL_FILE_PREFIX = "local:";
+/** `local:<localId>` → `<localId>`; null for any other value. */
+export declare function localUploadId(ref: unknown): string | null;
+/** Default cap on the bytes waiting in the queue: 200 MB. */
+export declare const DEFAULT_MAX_QUEUED_UPLOAD_BYTES: number;
+/** Thrown by `queueUpload` when the file would take the queue past its
+ *  size limit. Past the limit the browser may evict the app's storage,
+ *  which would lose every queued file, so the queue refuses instead. */
+export declare class UploadQueueFullError extends Error {
+    readonly code = "UPLOAD_QUEUE_FULL";
+    readonly limitBytes: number;
+    readonly queuedBytes: number;
+    constructor(limitBytes: number, queuedBytes: number, fileBytes: number);
+}
+export declare class MemoryUploadStore implements UploadStore {
+    private rows;
+    put(record: QueuedUpload): Promise<void>;
+    delete(localId: string): Promise<void>;
+    list(): Promise<QueuedUpload[]>;
+}
+/** Queued uploads in IndexedDB database `pylon_uploads_<appName>`. */
+export declare class IndexedDBUploadStore implements UploadStore {
+    private readonly appName;
+    private db;
+    constructor(appName?: string);
+    private open;
+    private run;
+    put(record: QueuedUpload): Promise<void>;
+    delete(localId: string): Promise<void>;
+    list(): Promise<QueuedUpload[]>;
+    /** Close the connection (tests; a deleteDatabase waits for it). */
+    close(): Promise<void>;
+}
+/** What the queue does with an upload, by its owner and who is signed in. */
+export type UploadOwnerVerdict = "send" | "hold" | "discard";
+/**
+ * The owner rule for uploads, the same as for queued writes: send when the
+ * upload's owner is signed in (or it was a guest's and that guest signed in
+ * to an account), hold while nobody or nobody known is signed in, discard
+ * when another user is signed in.
+ */
+export declare function uploadOwnerVerdict(owner: string | null, current: string | null | undefined): UploadOwnerVerdict;
+/** Whether an upload error is worth trying again later: no network, a
+ *  server error, a timeout, or a rate limit. Anything else (the type or
+ *  size was refused, a policy) is the server's answer. */
+export declare function isTransientUploadError(err: unknown): boolean;
+/** One queued upload as the outbox shows it. */
+export declare function uploadOutboxItem(r: QueuedUpload): OutboxItem;
+export declare function newUploadId(): string;
+/** What `queueUpload` returns. */
+export interface QueuedUploadHandle {
+    localId: string;
+    /** `local:<localId>`: the value the record's field holds until the
+     *  upload finishes. */
+    ref: string;
+    /** An object URL of the file, to show it at once. Null where the
+     *  platform has no `URL.createObjectURL`. */
+    localUrl: string | null;
+    /** Resolves with the uploaded file once it is attached; rejects with an
+     *  `UploadRejectedError` when the server refuses it or another user is
+     *  signed in. A page that reloads loses this promise, not the upload. */
+    done: Promise<UploadedFile>;
+}
+/** Why a queued upload was not sent. `code` is the server's error code
+ *  (`FILE_TOO_LARGE`, `POLICY_DENIED`, ...), or `DISCARDED` when another
+ *  user is signed in. */
+export declare class UploadRejectedError extends Error {
+    readonly code: string;
+    readonly localId: string;
+    constructor(message: string, code: string, localId: string);
+}
+/** What the upload manager needs from the sync engine. */
+export interface UploadManagerHost {
+    transport(): TransportConfig;
+    /** The signed-in user (`null` = anonymous), or `undefined` while it is
+     *  not known (no answer from the server yet this run, or a token change
+     *  in flight). */
+    currentOwner(): string | null | undefined;
+    /** The user a new upload belongs to; `undefined` = not known yet. */
+    ownerForNewUpload(): string | null | undefined;
+    /** Only the leader tab uploads. */
+    isLeader(): boolean;
+    isRunning(): boolean;
+    /** Queue an ordinary write that sets fields on a record. */
+    update(entity: string, id: string, patch: Record<string, unknown>): void;
+    /** The outbox changed. */
+    changed(): void;
+    /** Tell other tabs to reload the queue. */
+    broadcast(): void;
+    /** Run `fn` after `ms` unless the engine stops. */
+    later(ms: number, fn: () => void): void;
+}
+/**
+ * The offline upload queue: storage, the one-at-a-time upload loop, the
+ * owner rule, the size limit, and the outbox entries.
+ */
+export declare class UploadManager {
+    private readonly host;
+    private readonly maxBytes;
+    private store;
+    private records;
+    private objectUrls;
+    private waiters;
+    private running;
+    private failureCount;
+    private retryScheduled;
+    private _version;
+    constructor(host: UploadManagerHost, maxBytes?: number);
+    /** Bumped on every change to the queue. */
+    version(): number;
+    private changed;
+    /** Swap in persistent storage and load what it holds. */
+    attachStore(store: UploadStore): Promise<void>;
+    /** Re-read the queue from storage (another tab changed it). */
+    reload(): Promise<void>;
+    /** Queued uploads, oldest first. */
+    list(): QueuedUpload[];
+    get(localId: string): QueuedUpload | undefined;
+    /** Bytes waiting on the device. */
+    queuedBytes(): number;
+    enqueue(blob: Blob, opts: {
+        attach: UploadAttachTarget;
+        filename?: string;
+        contentType?: string;
+        visibility?: FileVisibility;
+    }): Promise<QueuedUploadHandle>;
+    /** An object URL for a queued file, or null. */
+    localUrl(localId: string): string | null;
+    private revoke;
+    private settle;
+    private remove;
+    /** Stamp uploads made before anyone was known to be signed in. */
+    stampPendingOwner(owner: string | null): Promise<void>;
+    /**
+     * Upload every queued file the signed-in user owns, oldest first, one at
+     * a time. Stops at the first error that says the network is down and
+     * tries again later (and on the next reconnect). A file the server
+     * refuses is marked failed and not tried again until `retry`.
+     */
+    process(): Promise<void>;
+    private scheduleRetry;
+    /** Queue a failed upload again. Throws `UploadRejectedError` with code
+     *  `OWNER_MISMATCH` / `OWNER_UNKNOWN` as `retryWrite` does. */
+    retry(localId: string): Promise<void>;
+    /** Drop a failed upload and its bytes. False when there is no failed
+     *  upload with that id. The record keeps its `local:` value. */
+    dismiss(localId: string): boolean;
+}

package/dist/upload.d.ts

@@ -0,0 +1,37 @@
+import { type TransportConfig } from "./transport";
+export interface UploadedFile {
+    id: string;
+    /** Where to read the file. Store this on your row. */
+    url: string;
+    size: number;
+}
+/**
+ * Who can read an uploaded file through `url`.
+ *   - "private" (default): only the uploader.
+ *   - "public": anyone, signed in or not. Use it for content every visitor
+ *     sees, such as feed photos or listing images.
+ * Writes and deletes stay with the uploader either way. On a CDN-backed
+ * store (Stack0, a public bucket) every file is served by its URL, so the
+ * setting only changes reads that go through `/api/files/<id>`.
+ */
+export type FileVisibility = "private" | "public";
+export interface UploadBytesOptions {
+    filename?: string;
+    contentType?: string;
+    visibility?: FileVisibility;
+}
+/**
+ * Upload a file (File, Blob, or raw bytes) in three steps:
+ *   1. POST /api/files/init with the name, type, size, and visibility;
+ *   2. PUT the bytes to the returned upload URL;
+ *   3. POST /api/files/confirm, which returns `{ id, url, size }`.
+ *
+ * The upload URL is either pylon's own receiver (local disk), which needs
+ * the session like any API call, or a presigned storage URL, whose
+ * signature is the credential. The session is never sent to a presigned
+ * URL on another origin.
+ *
+ * Throws `PylonHttpError` (with `status`) when the server answers with an
+ * error, and the fetch error when there is no network.
+ */
+export declare function uploadBytes(transport: TransportConfig, input: File | Blob | ArrayBuffer | Uint8Array, options?: UploadBytesOptions): Promise<UploadedFile>;

package/package.json

@@ -3,7 +3,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "0.22.47",
+  "version": "0.22.58",
   "type": "module",
   "main": "./src/index.ts",
   "types": "./dist/index.d.ts",

package/src/first-sync-requests.test.ts

@@ -0,0 +1,147 @@
+// A first sync is one snapshot. Measured on the Veritas build: after a
+// 12-page snapshot, the client fetched every entity a second time through
+// /api/entities/<E>/cursor (28 requests, 6.5 MB), because the connect
+// reconcile ran after a later delta pull reset the "started from zero"
+// flag, and empty observed entities each started their own reconcile.
+
+import { afterEach, expect, test } from "bun:test";
+import { createTestEnv, type TestEnv } from "./test-harness";
+
+let env: TestEnv | null = null;
+let restoreFetch: (() => void) | null = null;
+afterEach(async () => {
+  restoreFetch?.();
+  restoreFetch = null;
+  if (env) {
+    await env.dispose();
+    env = null;
+  }
+});
+
+function recordUrls(): string[] {
+  const urls: string[] = [];
+  const inner = globalThis.fetch;
+  globalThis.fetch = ((input: RequestInfo | URL, init?: RequestInit) => {
+    urls.push(typeof input === "string" ? input : input instanceof URL ? input.href : input.url);
+    return inner(input, init);
+  }) as typeof fetch;
+  restoreFetch = () => {
+    globalThis.fetch = inner;
+  };
+  return urls;
+}
+
+test("a cold start with a websocket makes no cursor requests after the snapshot", async () => {
+  env = await createTestEnv({ transport: "websocket" });
+  env.server.snapshotMode = true;
+  env.signIn({ userId: "u1" });
+  env.server.seed("Property", Array.from({ length: 50 }, (_, i) => ({ id: `p${i}` })));
+  env.server.seed("Visit", Array.from({ length: 20 }, (_, i) => ({ id: `v${i}` })));
+  const urls = recordUrls();
+  // Screens observe entities before and during the first sync, including
+  // ones with no rows.
+  env.engine.observeEntity("Property");
+  env.engine.observeEntity("Member");
+  await env.engine.start();
+  env.engine.observeEntity("Task");
+  await env.flush();
+  await env.engine.pull();
+  await env.flush();
+
+  expect(env.engine.store.list("Property").length).toBe(50);
+  expect(env.engine.store.list("Visit").length).toBe(20);
+  expect(urls.filter((u) => u.includes("/cursor"))).toEqual([]);
+});
+
+test("a reconnect within the interval skips the sweep; after it, the sweep runs", async () => {
+  env = await createTestEnv({ transport: "websocket", autoReconcileIntervalMs: 50, reconnectDelay: 5 });
+  env.server.snapshotMode = true;
+  env.signIn({ userId: "u1" });
+  env.server.seed("Property", [{ id: "p1" }]);
+  await env.engine.start();
+  await env.flush();
+  const urls = recordUrls();
+
+  env.transport.closeLatestWs();
+  await env.flush();
+  expect(urls.filter((u) => u.includes("/cursor"))).toEqual([]);
+
+  await new Promise((r) => setTimeout(r, 60));
+  env.transport.closeLatestWs();
+  await env.flush();
+  await env.flush(100);
+  expect(urls.some((u) => u.includes("/api/entities/Property/cursor"))).toBe(true);
+});
+
+/** A minimal `document` so the engine attaches its visibility listener. */
+function installDocument() {
+  const target = new EventTarget() as EventTarget & { visibilityState: string };
+  target.visibilityState = "visible";
+  const g = globalThis as { document?: unknown };
+  const previous = g.document;
+  g.document = target;
+  return {
+    foreground() {
+      target.dispatchEvent(new Event("visibilitychange"));
+    },
+    restore() {
+      if (previous === undefined) delete g.document;
+      else g.document = previous;
+    },
+  };
+}
+
+test("coming to the foreground does not sweep unless the app was offline", async () => {
+  const doc = installDocument();
+  try {
+    env = await createTestEnv({ transport: "websocket", autoReconcileIntervalMs: 20, reconnectDelay: 5 });
+    env.server.snapshotMode = true;
+    env.signIn({ userId: "u1" });
+    env.server.seed("Property", [{ id: "p1" }]);
+    await env.engine.start();
+    await env.flush();
+    const urls = recordUrls();
+
+    // Long past the interval, but the connection never dropped.
+    await new Promise((r) => setTimeout(r, 40));
+    doc.foreground();
+    await env.flush();
+    expect(urls.filter((u) => u.includes("/cursor"))).toEqual([]);
+
+    // The connection drops (a phone in a pocket), then the app comes back.
+    env.transport.closeLatestWs();
+    await env.flush(50);
+    doc.foreground();
+    await env.flush(50);
+    expect(urls.some((u) => u.includes("/api/entities/Property/cursor"))).toBe(true);
+
+    // Swept: the next foreground does not sweep again.
+    const swept = urls.length;
+    doc.foreground();
+    await env.flush();
+    expect(urls.slice(swept).filter((u) => u.includes("/cursor"))).toEqual([]);
+  } finally {
+    doc.restore();
+  }
+});
+
+test("a catch-up snapshot that could not remove deletes is followed by a sweep", async () => {
+  env = await createTestEnv({ transport: "websocket", reconnectDelay: 5 });
+  env.server.snapshotMode = true;
+  env.signIn({ userId: "u1" });
+  env.server.seed("Lead", [{ id: "a" }, { id: "b" }]);
+  await env.engine.start();
+  await env.flush();
+
+  // An older server: its snapshot does not list the entities it covers.
+  env.server.omitSnapshotEntities = true;
+  env.server.farBehind = () => true;
+  env.server.delete("Lead", "b");
+  const urls = recordUrls();
+  await env.engine.pull();
+  // The snapshot could not drop b.
+  env.transport.closeLatestWs();
+  await env.flush(50);
+  expect(urls.some((u) => u.includes("/api/entities/Lead/cursor"))).toBe(true);
+  expect(env.engine.store.list("Lead").map((r) => r.id)).toEqual(["a"]);
+});