@tensorix/metahub 0.1.11 → 0.2.1

package/dist/core/sync/storage-s3-bun.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"storage-s3-bun.d.ts","sourceRoot":"","sources":["../../../src/core/sync/storage-s3-bun.ts"],"names":[],"mappings":"AAUA,OAAO,EAML,KAAK,QAAQ,EACd,MAAM,cAAc,CAAC;AAiHtB,yEAAyE;AACzE,eAAO,MAAM,mBAAmB,SAAS,CAAC;AAE1C;;;;;GAKG;AACH,wBAAsB,UAAU,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAanG;AAED,+EAA+E;AAC/E,wBAAsB,aAAa,CAAC,MAAM,EAAE,QAAQ,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAS5E;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,OAAO,GAAG,MAAM,CAoB/F;AAED;;;;;;GAMG;AACH,wBAAsB,aAAa,CACjC,MAAM,EAAE,QAAQ,EAChB,OAAO,EAAE,MAAM,EAAE,EACjB,IAAI,GAAE;IAAE,KAAK,CAAC,EAAE,OAAO,CAAA;CAAO,GAC7B,OAAO,CAAC,IAAI,CAAC,CAmBf"}

package/dist/core/sync/storage-url.d.ts

@@ -0,0 +1,18 @@
+export declare const DEFAULT_STORAGE_PREFIX = "metahub";
+/**
+ * Stable host[:port] component of an S3 endpoint for the peer key. The default
+ * port is stripped so `https://h` and `https://h:443` collapse to one key (that
+ * spurious split is exactly the collision class the endpoint-qualified key is
+ * meant to remove). Falls back to a parsed-by-hand host when the endpoint isn't
+ * a valid URL, kept deterministic.
+ */
+export declare function storageEndpointHost(endpoint: string): string;
+/**
+ * Synthetic peer key for a storage peer: one per endpoint+bucket+prefix. The
+ * endpoint host is part of the identity because bucket names are only unique
+ * within a single S3-compatible endpoint (R2/MinIO/COS each have their own
+ * namespace) — without it, the same bucket+prefix on a second endpoint would
+ * collide with (and silently overwrite, keeping the stale cursor) the first.
+ */
+export declare function storageUrl(endpoint: string, bucket: string, prefix?: string): string;
+//# sourceMappingURL=storage-url.d.ts.map

package/dist/core/sync/storage-url.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"storage-url.d.ts","sourceRoot":"","sources":["../../../src/core/sync/storage-url.ts"],"names":[],"mappings":"AASA,eAAO,MAAM,sBAAsB,YAAY,CAAC;AAIhD;;;;;;GAMG;AACH,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAY5D;AAED;;;;;;GAMG;AACH,wBAAgB,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,MAAM,CAGpF"}

package/dist/core/sync/storage.d.ts

@@ -0,0 +1,170 @@
+import type { DbDriver } from "../driver.ts";
+import type { SyncResult } from "./client.ts";
+/** An 's3' peer's bucket settings, stored as JSON in peers.config (local-only). */
+export interface S3Config {
+    endpoint: string;
+    region: string;
+    bucket: string;
+    prefix: string;
+    accessKeyId: string;
+    secretAccessKey: string;
+    /** When false (`--no-encrypt`) segments are gzip-only plaintext. */
+    encrypt: boolean;
+    /** base64 raw 32-byte master key; present iff encrypt. Resolved at peer-add. */
+    masterKey?: string;
+    /**
+     * Address objects as `<bucket>.<host>/<key>` (virtual-hosted) instead of
+     * `<host>/<bucket>/<key>` (path-style). Some providers (Tencent COS) reject
+     * path-style outright. When undefined, the clients auto-detect: virtual-hosted
+     * iff the endpoint host already starts with `<bucket>.` (see
+     * isVirtualHostedStyle). R2/MinIO keep using path-style.
+     */
+    virtualHostedStyle?: boolean;
+    /**
+     * Node-role hints for this bucket (not bucket settings — safe to omit from
+     * shared enroll links). `publish` marks this node as the bucket's publisher
+     * (writes whole-hub snapshots; see StorageSyncOpts.publish): true for the data
+     * home (a server, or a no-origin PWA which IS the home), false for a replica
+     * that only attaches the bucket for its own away-sync. `priority` ranks
+     * publisher candidates when several are eligible (higher wins; see
+     * publisher-lease.ts) — server > desktop > laptop > phone.
+     */
+    publish?: boolean;
+    priority?: number;
+}
+/** Whether to use virtual-hosted addressing for this bucket: the explicit flag,
+ *  or — when unset — auto-detected from an endpoint whose host already carries
+ *  the bucket (e.g. COS's `<bucket>.cos.<region>.myqcloud.com`). */
+export declare function isVirtualHostedStyle(config: S3Config): boolean;
+export interface StorageObject {
+    key: string;
+    etag?: string;
+}
+/** Options for a conditional/typed put. */
+export interface StoragePutOpts {
+    contentType?: string;
+    /** Conditional create (S3 `If-None-Match: *`): fail with an MhError("conflict")
+     *  if the object already exists. Used to make first-time key provisioning a
+     *  compare-and-set so two devices initializing the same empty bucket can't
+     *  clobber each other's master key. */
+    ifNoneMatch?: boolean;
+}
+/** The minimal object-store surface storage-sync needs. Implemented per runtime:
+ *  Bun.S3Client on the CLI/desktop (storage-s3-bun.ts), aws4fetch SigV4 + fetch
+ *  in the browser worker (webui/data/storage-s3-browser.ts). */
+export interface StorageClient {
+    /** Keys under `prefix`, ascending, optionally only those strictly after
+     *  `startAfter`. With `delimiter` ("/"), collapse one level into common
+     *  prefixes (their keys end in the delimiter) to discover child "folders"
+     *  cheaply — callers must tolerate a backend that ignores it and returns full
+     *  keys instead. */
+    list(prefix: string, startAfter?: string, delimiter?: string): Promise<StorageObject[]>;
+    get(key: string): Promise<Uint8Array | null>;
+    put(key: string, body: Uint8Array, opts?: StoragePutOpts): Promise<void>;
+    del(key: string): Promise<void>;
+}
+type StorageClientFactory = (config: S3Config) => StorageClient;
+/** Register this runtime's S3 client builder (see the per-runtime impls). */
+export declare function setStorageClientFactory(f: StorageClientFactory): void;
+export declare function storageClientFor(config: S3Config): StorageClient;
+/** Upload a blob's bytes to the bucket (idempotent via If-None-Match). Returns
+ *  true when actually written, false when the object already existed. */
+export declare function putBucketBlob(config: S3Config, hash: string, bytes: Uint8Array): Promise<boolean>;
+/** Every blob hash the bucket durably holds — one paginated LIST of the `blobs/`
+ *  namespace (the client loops continuation tokens internally). Used by the
+ *  on-demand clear-presence verify; keeps the `<base>/blobs/<hash>` key layout
+ *  encapsulated here. Throws if the bucket is unreachable (caller treats the
+ *  anchor as unverified). */
+export declare function listBucketBlobHashes(config: S3Config): Promise<Set<string>>;
+/** Fetch a blob's bytes from the bucket, or null if absent. */
+export declare function getBucketBlob(config: S3Config, hash: string): Promise<Uint8Array | null>;
+/**
+ * Resolve the master key for a bucket at setup: read keys/main.json and unwrap
+ * with the passphrase, or — when absent — generate a key, wrap it, and upload
+ * it with `If-None-Match` so a concurrent first-time init can't clobber it.
+ * Returns base64(K) to store in peers.config, or null when encryption is off.
+ * Wrong passphrase throws `auth` (from unwrapMasterKey).
+ */
+export declare function provisionMasterKey(client: StorageClient, config: S3Config, passphrase: string): Promise<string | null>;
+/**
+ * Publish a winners-only snapshot and truncate this node's own segments. Every
+ * op we've uploaded so far is in the local oplog now, so the snapshot (winners
+ * as-of the current max HLC) supersedes all of them — they're safe to delete
+ * (compact.ts invariant #1). Ops produced afterwards go into fresh, higher-keyed
+ * segments; consumers reach our pre-snapshot state via the snapshot itself.
+ *
+ * The key is `<maxHlc>~<hash>.snap`: the hash (over the canonical winner set)
+ * makes converged nodes collide on one key (deduped) while two nodes that are
+ * NOT converged — same max HLC but different extra winners — get distinct keys,
+ * so neither overwrites the other. Then old snapshots (strictly lower max-HLC)
+ * are GC'd so the bucket stays bounded.
+ *
+ * Returns the snapshot key + change count, or null when the oplog is empty.
+ */
+export declare function publishSnapshot(db: DbDriver, client: StorageClient, config: S3Config, retainSegments?: number): Promise<{
+    key: string;
+    changes: number;
+} | null>;
+export interface StorageSyncOpts {
+    /** Publisher-only: publish a snapshot + truncate once this node's own segment
+     *  count reaches this. Default 200 — high enough that typical use rarely hits
+     *  it, low enough to keep the bucket bounded under heavy publisher use. */
+    snapshotEverySegments?: number;
+    /** Push-batching: defer uploading our pending own ops until there are at least
+     *  this many (default 1 → push every round). Coalesces a burst of edits into
+     *  one segment instead of many tiny ones (each is a billed request + a GET for
+     *  every puller). */
+    minPushChanges?: number;
+    /** …or until the oldest pending op is at least this old (ms; default 0 → no
+     *  age bound). Bounds how long a small trailing edit waits. */
+    maxPushAgeMs?: number;
+    /** Push regardless of the batching thresholds — use on explicit "sync now" and
+     *  when the app is backgrounding, so edits are never stranded unsynced. */
+    forcePush?: boolean;
+    /**
+     * This node is the bucket's *publisher*: keep the bucket a complete, fresh
+     * whole-hub mirror by writing a winners-only snapshot (publishSnapshot) when
+     * the bucket has none yet, or when the hub has advanced past the newest
+     * snapshot. The snapshot spans the WHOLE db, so it captures data authored by
+     * nodes that never push to this bucket themselves (e.g. a server holding the
+     * canonical hub while browsers only window onto it) — fixing the "attached a
+     * bucket but it stays empty" footgun. Any node holding the full hub can be the
+     * publisher; who is one is decided above storage.ts (server by default, else a
+     * lease — see publisher-lease.ts). Correctness never depends on a single
+     * publisher: concurrent snapshots are content-addressed and idempotent. */
+    publish?: boolean;
+    /** ⑤d snapshot trigger: publish once the whole-hub delta since the newest
+     *  snapshot reaches max(`snapshotMinDelta` ops, hub_size × `snapshotDeltaRatio`),
+     *  or `snapshotMaxIntervalMs` elapsed (safety). Replaces the old 60s timer that
+     *  re-checkpointed constantly and forced full re-downloads. Defaults:
+     *  DEFAULT_SNAPSHOT_MIN_DELTA / _DELTA_RATIO / _MAX_INTERVAL_MS. */
+    snapshotMinDelta?: number;
+    snapshotDeltaRatio?: number;
+    snapshotMaxIntervalMs?: number;
+    /** ⑤c: keep this many of our most recent own segments alive past a snapshot so
+     *  slightly-behind consumers catch up incrementally (default DEFAULT_RETAIN_SEGMENTS). */
+    snapshotRetainSegments?: number;
+    /** This node's publisher priority when contending for the lease (higher wins;
+     *  see publisher-lease.ts). Only consulted when `publish` is set. */
+    priority?: number;
+    /**
+     * Run the PULL half of the round (snapshots + other nodes' segments). Default
+     * true. Set false for *push-only* rounds: an origin-backed replica gets a
+     * superset of bucket contents from its origin (HTTP), so it never needs to PULL
+     * the bucket — it only PUSHes its own segments here for independent durability
+     * (Path B). Skipping PULL drops the two unconditional LISTs (snapshot/, oplog/)
+     * that dominate idle bucket cost. A *publisher* must always PULL (it needs the
+     * snapshot list to decide what to publish), so callers never combine
+     * `publish:true` with `pull:false`.
+     */
+    pull?: boolean;
+}
+/**
+ * One push/pull round against a storage peer. Push uploads this node's pending
+ * own ops as a new segment (subject to batching); pull hydrates from snapshots
+ * (any not yet ingested) then drains each other node's new segments. Returns
+ * { pushed, pulled } like syncWithPeer so peers.ts status writeback is shared.
+ */
+export declare function syncWithStorage(db: DbDriver, peerUrl: string, client: StorageClient, config: S3Config, opts?: StorageSyncOpts): Promise<SyncResult>;
+export {};
+//# sourceMappingURL=storage.d.ts.map

package/dist/core/sync/storage.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"storage.d.ts","sourceRoot":"","sources":["../../../src/core/sync/storage.ts"],"names":[],"mappings":"AAmBA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AAK7C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAc9C,mFAAmF;AACnF,MAAM,WAAW,QAAQ;IACvB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,MAAM,CAAC;IACxB,oEAAoE;IACpE,OAAO,EAAE,OAAO,CAAC;IACjB,gFAAgF;IAChF,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;;;OAMG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;oEAEoE;AACpE,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,QAAQ,GAAG,OAAO,CAO9D;AAED,MAAM,WAAW,aAAa;IAC5B,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,2CAA2C;AAC3C,MAAM,WAAW,cAAc;IAC7B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;2CAGuC;IACvC,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB;AAED;;gEAEgE;AAChE,MAAM,WAAW,aAAa;IAC5B;;;;wBAIoB;IACpB,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC;IACxF,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC;IAC7C,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACzE,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACjC;AAED,KAAK,oBAAoB,GAAG,CAAC,MAAM,EAAE,QAAQ,KAAK,aAAa,CAAC;AAGhE,6EAA6E;AAC7E,wBAAgB,uBAAuB,CAAC,CAAC,EAAE,oBAAoB,GAAG,IAAI,CAErE;AAED,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,QAAQ,GAAG,aAAa,CAIhE;AAmGD;yEACyE;AACzE,wBAAsB,aAAa,CACjC,MAAM,EAAE,QAAQ,EAChB,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,UAAU,GAChB,OAAO,CAAC,OAAO,CAAC,CAYlB;AAED;;;;6BAI6B;AAC7B,wBAAsB,oBAAoB,CAAC,MAAM,EAAE,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAUjF;AAED,+DAA+D;AAC/D,wBAAsB,aAAa,CAAC,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,CAM9F;AA8ID;;;;;;GAMG;AACH,wBAAsB,kBAAkB,CACtC,MAAM,EAAE,aAAa,EACrB,MAAM,EAAE,QAAQ,EAChB,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CA2BxB;AAID;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,eAAe,CACnC,EAAE,EAAE,QAAQ,EACZ,MAAM,EAAE,aAAa,EACrB,MAAM,EAAE,QAAQ,EAChB,cAAc,GAAE,MAAgC,GAC/C,OAAO,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAAC,CAkDlD;AAID,MAAM,WAAW,eAAe;IAC9B;;+EAE2E;IAC3E,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B;;;yBAGqB;IACrB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;mEAC+D;IAC/D,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;+EAC2E;IAC3E,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;;;;;;;;;;+EAU2E;IAC3E,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB;;;;wEAIoE;IACpE,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B;8FAC0F;IAC1F,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC;yEACqE;IACrE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;;;;;;;OASG;IACH,IAAI,CAAC,EAAE,OAAO,CAAC;CAChB;AAwBD;;;;;GAKG;AACH,wBAAsB,eAAe,CACnC,EAAE,EAAE,QAAQ,EACZ,OAAO,EAAE,MAAM,EACf,MAAM,EAAE,aAAa,EACrB,MAAM,EAAE,QAAQ,EAChB,IAAI,GAAE,eAAoB,GACzB,OAAO,CAAC,UAAU,CAAC,CAiIrB"}