@atproto/repo 0.0.1 → 0.1.0

package/dist/mst/diff.d.ts

@@ -1,33 +1,4 @@
-import * as auth from '@atproto/auth';
-import { CID } from 'multiformats';
-import CidSet from '../cid-set';
-export declare class DataDiff {
-    adds: Record<string, DataAdd>;
-    updates: Record<string, DataUpdate>;
-    deletes: Record<string, DataDelete>;
-    newCids: CidSet;
-    recordAdd(key: string, cid: CID): void;
-    recordUpdate(key: string, prev: CID, cid: CID): void;
-    recordDelete(key: string, cid: CID): void;
-    recordNewCid(cid: CID): void;
-    addDiff(diff: DataDiff): void;
-    addList(): DataAdd[];
-    updateList(): DataUpdate[];
-    deleteList(): DataDelete[];
-    newCidList(): CID[];
-    updatedKeys(): string[];
-    neededCapabilities(rootDid: string): auth.ucans.Capability[];
-}
-export declare type DataAdd = {
-    key: string;
-    cid: CID;
-};
-export declare type DataUpdate = {
-    key: string;
-    prev: CID;
-    cid: CID;
-};
-export declare type DataDelete = {
-    key: string;
-    cid: CID;
-};
+import { DataDiff } from '../data-diff';
+import MST from './mst';
+export declare const nullDiff: (tree: MST) => Promise<DataDiff>;
+export declare const mstDiff: (curr: MST, prev: MST | null) => Promise<DataDiff>;

package/dist/mst/mst.d.ts

@@ -1,74 +1,112 @@
 import z from 'zod';
 import { CID } from 'multiformats';
-import IpldStore from '../blockstore/ipld-store';
-import { DataDiff } from './diff';
+import { ReadableBlockstore } from '../storage';
 import { DataStore } from '../types';
 import { BlockWriter } from '@ipld/car/api';
-export declare const nodeDataDef: z.ZodObject<{
+import BlockMap from '../block-map';
+import CidSet from '../cid-set';
+declare const nodeData: z.ZodObject<{
     l: z.ZodNullable<z.ZodEffects<z.ZodEffects<z.ZodAny, any, any>, CID, any>>;
     e: z.ZodArray<z.ZodObject<{
         p: z.ZodNumber;
-        k: z.ZodString;
+        k: z.ZodType<Uint8Array, z.ZodTypeDef, Uint8Array>;
         v: z.ZodEffects<z.ZodEffects<z.ZodAny, any, any>, CID, any>;
         t: z.ZodNullable<z.ZodEffects<z.ZodEffects<z.ZodAny, any, any>, CID, any>>;
     }, "strip", z.ZodTypeAny, {
-        t: CID | null;
         p: number;
-        k: string;
+        k: Uint8Array;
         v: CID;
+        t: CID | null;
     }, {
-        t?: any;
         v?: any;
+        t?: any;
         p: number;
-        k: string;
+        k: Uint8Array;
     }>, "many">;
 }, "strip", z.ZodTypeAny, {
     l: CID | null;
     e: {
-        t: CID | null;
         p: number;
-        k: string;
+        k: Uint8Array;
         v: CID;
+        t: CID | null;
     }[];
 }, {
     l?: any;
     e: {
-        t?: any;
         v?: any;
+        t?: any;
         p: number;
-        k: string;
+        k: Uint8Array;
     }[];
 }>;
-export declare type NodeData = z.infer<typeof nodeDataDef>;
+export declare type NodeData = z.infer<typeof nodeData>;
+export declare const nodeDataDef: {
+    name: string;
+    schema: z.ZodObject<{
+        l: z.ZodNullable<z.ZodEffects<z.ZodEffects<z.ZodAny, any, any>, CID, any>>;
+        e: z.ZodArray<z.ZodObject<{
+            p: z.ZodNumber;
+            k: z.ZodType<Uint8Array, z.ZodTypeDef, Uint8Array>;
+            v: z.ZodEffects<z.ZodEffects<z.ZodAny, any, any>, CID, any>;
+            t: z.ZodNullable<z.ZodEffects<z.ZodEffects<z.ZodAny, any, any>, CID, any>>;
+        }, "strip", z.ZodTypeAny, {
+            p: number;
+            k: Uint8Array;
+            v: CID;
+            t: CID | null;
+        }, {
+            v?: any;
+            t?: any;
+            p: number;
+            k: Uint8Array;
+        }>, "many">;
+    }, "strip", z.ZodTypeAny, {
+        l: CID | null;
+        e: {
+            p: number;
+            k: Uint8Array;
+            v: CID;
+            t: CID | null;
+        }[];
+    }, {
+        l?: any;
+        e: {
+            v?: any;
+            t?: any;
+            p: number;
+            k: Uint8Array;
+        }[];
+    }>;
+};
 export declare type NodeEntry = MST | Leaf;
-export declare type Fanout = 2 | 8 | 16 | 32 | 64;
 export declare type MstOpts = {
     layer: number;
-    fanout: Fanout;
 };
 export declare class MST implements DataStore {
-    blockstore: IpldStore;
-    fanout: Fanout;
+    storage: ReadableBlockstore;
     entries: NodeEntry[] | null;
     layer: number | null;
     pointer: CID;
     outdatedPointer: boolean;
-    constructor(blockstore: IpldStore, fanout: Fanout, pointer: CID, entries: NodeEntry[] | null, layer: number | null);
-    static create(blockstore: IpldStore, entries?: NodeEntry[], opts?: Partial<MstOpts>): Promise<MST>;
-    static fromData(blockstore: IpldStore, data: NodeData, opts?: Partial<MstOpts>): Promise<MST>;
-    static load(blockstore: IpldStore, cid: CID, opts?: Partial<MstOpts>): MST;
+    constructor(storage: ReadableBlockstore, pointer: CID, entries: NodeEntry[] | null, layer: number | null);
+    static create(storage: ReadableBlockstore, entries?: NodeEntry[], opts?: Partial<MstOpts>): Promise<MST>;
+    static fromData(storage: ReadableBlockstore, data: NodeData, opts?: Partial<MstOpts>): Promise<MST>;
+    static load(storage: ReadableBlockstore, cid: CID, opts?: Partial<MstOpts>): MST;
     newTree(entries: NodeEntry[]): Promise<MST>;
     getEntries(): Promise<NodeEntry[]>;
     getPointer(): Promise<CID>;
     getLayer(): Promise<number>;
     attemptGetLayer(): Promise<number | null>;
-    stage(): Promise<CID>;
-    stageRecurse(trimTop?: boolean): Promise<CID>;
+    getUnstoredBlocks(): Promise<{
+        root: CID;
+        blocks: BlockMap;
+    }>;
     add(key: string, value: CID, knownZeros?: number): Promise<MST>;
     get(key: string): Promise<CID | null>;
     update(key: string, value: CID): Promise<MST>;
     delete(key: string): Promise<MST>;
-    diff(other: MST): Promise<DataDiff>;
+    deleteRecurse(key: string): Promise<MST>;
     updateEntry(index: number, entry: NodeEntry): Promise<MST>;
     removeEntry(index: number): Promise<MST>;
     append(entry: NodeEntry): Promise<MST>;
@@ -77,20 +115,25 @@ export declare class MST implements DataStore {
     slice(start?: number | undefined, end?: number | undefined): Promise<NodeEntry[]>;
     spliceIn(entry: NodeEntry, index: number): Promise<MST>;
     replaceWithSplit(index: number, left: MST | null, leaf: Leaf, right: MST | null): Promise<MST>;
+    trimTop(): Promise<MST>;
     splitAround(key: string): Promise<[MST | null, MST | null]>;
     appendMerge(toMerge: MST): Promise<MST>;
     createChild(): Promise<MST>;
     createParent(): Promise<MST>;
     findGtOrEqualLeafIndex(key: string): Promise<number>;
     walkLeavesFrom(key: string): AsyncIterable<Leaf>;
-    list(count: number, after?: string, before?: string): Promise<Leaf[]>;
+    list(count?: number, after?: string, before?: string): Promise<Leaf[]>;
     listWithPrefix(prefix: string, count?: number): Promise<Leaf[]>;
     walk(): AsyncIterable<NodeEntry>;
     paths(): Promise<NodeEntry[][]>;
     allNodes(): Promise<NodeEntry[]>;
+    allCids(): Promise<CidSet>;
     leaves(): Promise<Leaf[]>;
     leafCount(): Promise<number>;
+    walkReachable(): AsyncIterable<NodeEntry>;
+    reachableLeaves(): Promise<Leaf[]>;
     writeToCarStream(car: BlockWriter): Promise<void>;
+    cidsForPath(key: string): Promise<CID[]>;
     isTree(): this is MST;
     isLeaf(): this is Leaf;
     equals(other: NodeEntry): Promise<boolean>;

package/dist/mst/util.d.ts

@@ -1,9 +1,17 @@
 import { CID } from 'multiformats';
-import IpldStore from '../blockstore/ipld-store';
-import { NodeEntry, NodeData, MstOpts, Fanout } from './mst';
-export declare const leadingZerosOnHash: (key: string, fanout: Fanout) => Promise<number>;
-export declare const layerForEntries: (entries: NodeEntry[], fanout: Fanout) => Promise<number | null>;
-export declare const deserializeNodeData: (blockstore: IpldStore, data: NodeData, opts?: Partial<MstOpts>) => Promise<NodeEntry[]>;
+import { ReadableBlockstore } from '../storage';
+import { NodeEntry, NodeData, MstOpts } from './mst';
+export declare const leadingZerosOnHash: (key: string | Uint8Array) => Promise<number>;
+export declare const layerForEntries: (entries: NodeEntry[]) => Promise<number | null>;
+export declare const deserializeNodeData: (storage: ReadableBlockstore, data: NodeData, opts?: Partial<MstOpts>) => Promise<NodeEntry[]>;
 export declare const serializeNodeData: (entries: NodeEntry[]) => NodeData;
 export declare const countPrefixLen: (a: string, b: string) => number;
 export declare const cidForEntries: (entries: NodeEntry[]) => Promise<CID>;
+export declare const isValidMstKey: (str: string) => boolean;
+export declare const validCharsRegex: RegExp;
+export declare const isValidChars: (str: string) => boolean;
+export declare const ensureValidMstKey: (str: string) => void;
+export declare class InvalidMstKeyError extends Error {
+    key: string;
+    constructor(key: string);
+}

package/dist/parse.d.ts

@@ -0,0 +1,16 @@
+import { check } from '@atproto/common';
+import { RepoRecord } from '@atproto/lexicon';
+import { CID } from 'multiformats/cid';
+import BlockMap from './block-map';
+export declare const getAndParseRecord: (blocks: BlockMap, cid: CID) => Promise<{
+    record: RepoRecord;
+    bytes: Uint8Array;
+}>;
+export declare const getAndParseByDef: <T>(blocks: BlockMap, cid: CID, def: check.Def<T>) => Promise<{
+    obj: T;
+    bytes: Uint8Array;
+}>;
+export declare const parseObjByDef: <T>(bytes: Uint8Array, cid: CID, def: check.Def<T>) => {
+    obj: T;
+    bytes: Uint8Array;
+};

package/dist/readable-repo.d.ts

@@ -0,0 +1,22 @@
+import { CID } from 'multiformats/cid';
+import { Commit, DataStore, RepoContents } from './types';
+import { ReadableBlockstore } from './storage';
+declare type Params = {
+    storage: ReadableBlockstore;
+    data: DataStore;
+    commit: Commit;
+    cid: CID;
+};
+export declare class ReadableRepo {
+    storage: ReadableBlockstore;
+    data: DataStore;
+    commit: Commit;
+    cid: CID;
+    constructor(params: Params);
+    static load(storage: ReadableBlockstore, commitCid: CID): Promise<ReadableRepo>;
+    get did(): string;
+    get version(): number;
+    getRecord(collection: string, rkey: string): Promise<unknown | null>;
+    getContents(): Promise<RepoContents>;
+}
+export default ReadableRepo;

package/dist/repo.d.ts

@@ -1,39 +1,23 @@
 import { CID } from 'multiformats/cid';
-import { BlockWriter } from '@ipld/car/writer';
-import { RepoRoot, Commit, DataStore, RepoMeta, RecordCreateOp, RecordWriteOp } from './types';
-import IpldStore from './blockstore/ipld-store';
-import * as auth from '@atproto/auth';
+import * as crypto from '@atproto/crypto';
+import { Commit, DataStore, RecordCreateOp, RecordWriteOp, CommitData } from './types';
+import { RepoStorage } from './storage';
+import { ReadableRepo } from './readable-repo';
 declare type Params = {
-    blockstore: IpldStore;
+    storage: RepoStorage;
     data: DataStore;
     commit: Commit;
-    root: RepoRoot;
-    meta: RepoMeta;
     cid: CID;
-    stagedWrites: RecordWriteOp[];
 };
-export declare class Repo {
-    blockstore: IpldStore;
-    data: DataStore;
-    commit: Commit;
-    root: RepoRoot;
-    meta: RepoMeta;
-    cid: CID;
-    stagedWrites: RecordWriteOp[];
+export declare class Repo extends ReadableRepo {
+    storage: RepoStorage;
     constructor(params: Params);
-    static create(blockstore: IpldStore, did: string, authStore: auth.AuthStore, initialRecords?: RecordCreateOp[]): Promise<Repo>;
-    static load(blockstore: IpldStore, cid: CID): Promise<Repo>;
-    private updateRepo;
-    get did(): string;
-    getRecord(collection: string, rkey: string): Promise<unknown | null>;
-    stageUpdate(write: RecordWriteOp | RecordWriteOp[]): Repo;
-    createCommit(authStore: auth.AuthStore, performUpdate?: (prev: CID, curr: CID) => Promise<CID | null>): Promise<Repo>;
-    revert(count: number): Promise<Repo>;
-    getCarNoHistory(): Promise<Uint8Array>;
-    getDiffCar(to: CID | null): Promise<Uint8Array>;
-    getFullHistory(): Promise<Uint8Array>;
-    private openCar;
-    writeCheckoutToCarStream(car: BlockWriter): Promise<void>;
-    writeCommitsToCarStream(car: BlockWriter, oldestCommit: CID | null, recentCommit: CID): Promise<void>;
+    static formatInitCommit(storage: RepoStorage, did: string, keypair: crypto.Keypair, initialWrites?: RecordCreateOp[]): Promise<CommitData>;
+    static createFromCommit(storage: RepoStorage, commit: CommitData): Promise<Repo>;
+    static create(storage: RepoStorage, did: string, keypair: crypto.Keypair, initialWrites?: RecordCreateOp[]): Promise<Repo>;
+    static load(storage: RepoStorage, cid?: CID): Promise<Repo>;
+    formatCommit(toWrite: RecordWriteOp | RecordWriteOp[], keypair: crypto.Keypair): Promise<CommitData>;
+    applyCommit(commitData: CommitData): Promise<Repo>;
+    applyWrites(toWrite: RecordWriteOp | RecordWriteOp[], keypair: crypto.Keypair): Promise<Repo>;
 }
 export default Repo;

package/dist/storage/index.d.ts

@@ -1 +1,5 @@
+export * from './readable-blockstore';
+export * from './repo-storage';
+export * from './memory-blockstore';
+export * from './sync-storage';
 export * from './types';

package/dist/storage/memory-blockstore.d.ts

@@ -0,0 +1,28 @@
+import { CID } from 'multiformats/cid';
+import { CommitData } from '../types';
+import BlockMap from '../block-map';
+import RepoStorage from './repo-storage';
+export declare class MemoryBlockstore extends RepoStorage {
+    blocks: BlockMap;
+    head: CID | null;
+    constructor(blocks?: BlockMap);
+    getHead(): Promise<CID | null>;
+    getBytes(cid: CID): Promise<Uint8Array | null>;
+    has(cid: CID): Promise<boolean>;
+    getBlocks(cids: CID[]): Promise<{
+        blocks: BlockMap;
+        missing: CID[];
+    }>;
+    putBlock(cid: CID, block: Uint8Array): Promise<void>;
+    putMany(blocks: BlockMap): Promise<void>;
+    indexCommits(commits: CommitData[]): Promise<void>;
+    updateHead(cid: CID, _prev: CID | null): Promise<void>;
+    applyCommit(commit: CommitData): Promise<void>;
+    getCommitPath(latest: CID, earliest: CID | null): Promise<CID[] | null>;
+    getBlocksForCommits(commits: CID[]): Promise<{
+        [commit: string]: BlockMap;
+    }>;
+    sizeInBytes(): Promise<number>;
+    destroy(): Promise<void>;
+}
+export default MemoryBlockstore;

package/dist/storage/readable-blockstore.d.ts

@@ -0,0 +1,24 @@
+import { check } from '@atproto/common';
+import { RepoRecord } from '@atproto/lexicon';
+import { CID } from 'multiformats/cid';
+import BlockMap from '../block-map';
+export declare abstract class ReadableBlockstore {
+    abstract getBytes(cid: CID): Promise<Uint8Array | null>;
+    abstract has(cid: CID): Promise<boolean>;
+    abstract getBlocks(cids: CID[]): Promise<{
+        blocks: BlockMap;
+        missing: CID[];
+    }>;
+    attemptRead<T>(cid: CID, def: check.Def<T>): Promise<{
+        obj: T;
+        bytes: Uint8Array;
+    } | null>;
+    readObjAndBytes<T>(cid: CID, def: check.Def<T>): Promise<{
+        obj: T;
+        bytes: Uint8Array;
+    }>;
+    readObj<T>(cid: CID, def: check.Def<T>): Promise<T>;
+    attemptReadRecord(cid: CID): Promise<RepoRecord | null>;
+    readRecord(cid: CID): Promise<RepoRecord>;
+}
+export default ReadableBlockstore;

package/dist/storage/repo-storage.d.ts

@@ -0,0 +1,18 @@
+import { CID } from 'multiformats/cid';
+import BlockMap from '../block-map';
+import { CommitBlockData, CommitData } from '../types';
+import ReadableBlockstore from './readable-blockstore';
+export declare abstract class RepoStorage extends ReadableBlockstore {
+    abstract getHead(forUpdate?: boolean): Promise<CID | null>;
+    abstract getCommitPath(latest: CID, earliest: CID | null): Promise<CID[] | null>;
+    abstract getBlocksForCommits(commits: CID[]): Promise<{
+        [commit: string]: BlockMap;
+    }>;
+    abstract putBlock(cid: CID, block: Uint8Array): Promise<void>;
+    abstract putMany(blocks: BlockMap): Promise<void>;
+    abstract updateHead(cid: CID, prev: CID | null): Promise<void>;
+    abstract indexCommits(commit: CommitData[]): Promise<void>;
+    applyCommit(commit: CommitData): Promise<void>;
+    getCommits(latest: CID, earliest: CID | null): Promise<CommitBlockData[] | null>;
+}
+export default RepoStorage;

package/dist/storage/sync-storage.d.ts

@@ -0,0 +1,15 @@
+import { CID } from 'multiformats/cid';
+import BlockMap from '../block-map';
+import ReadableBlockstore from './readable-blockstore';
+export declare class SyncStorage extends ReadableBlockstore {
+    staged: ReadableBlockstore;
+    saved: ReadableBlockstore;
+    constructor(staged: ReadableBlockstore, saved: ReadableBlockstore);
+    getBytes(cid: CID): Promise<Uint8Array | null>;
+    getBlocks(cids: CID[]): Promise<{
+        blocks: BlockMap;
+        missing: CID[];
+    }>;
+    has(cid: CID): Promise<boolean>;
+}
+export default SyncStorage;

package/dist/storage/types.d.ts

@@ -5,8 +5,11 @@ export interface BlobStore {
     putTemp(bytes: Uint8Array | stream.Readable): Promise<string>;
     makePermanent(key: string, cid: CID): Promise<void>;
     putPermanent(cid: CID, bytes: Uint8Array | stream.Readable): Promise<void>;
+    quarantine(cid: CID): Promise<void>;
+    unquarantine(cid: CID): Promise<void>;
     getBytes(cid: CID): Promise<Uint8Array>;
     getStream(cid: CID): Promise<stream.Readable>;
+    delete(cid: CID): Promise<void>;
 }
 export declare class BlobNotFoundError extends Error {
 }

package/dist/sync/consumer.d.ts

@@ -0,0 +1,18 @@
+import { CID } from 'multiformats/cid';
+import { RepoStorage } from '../storage';
+import Repo from '../repo';
+import * as verify from '../verify';
+import { RepoContents, WriteLog } from '../types';
+export declare const loadCheckout: (storage: RepoStorage, repoCar: Uint8Array, did: string, signingKey: string) => Promise<{
+    root: CID;
+    contents: RepoContents;
+}>;
+export declare const loadFullRepo: (storage: RepoStorage, repoCar: Uint8Array, did: string, signingKey: string) => Promise<{
+    root: CID;
+    writeLog: WriteLog;
+}>;
+export declare const loadDiff: (repo: Repo, diffCar: Uint8Array, did: string, signingKey: string) => Promise<{
+    root: CID;
+    writeLog: WriteLog;
+}>;
+export declare const persistUpdates: (storage: RepoStorage, updateStorage: RepoStorage, updates: verify.VerifiedUpdate[]) => Promise<WriteLog>;

package/dist/sync/index.d.ts

@@ -0,0 +1,2 @@
+export * from './consumer';
+export * from './provider';

package/dist/sync/provider.d.ts

@@ -0,0 +1,9 @@
+import { RecordPath } from '../types';
+import { BlockWriter } from '@ipld/car/writer';
+import { CID } from 'multiformats/cid';
+import { RepoStorage } from '../storage';
+export declare const getCheckout: (storage: RepoStorage, commitCid: CID) => Promise<Uint8Array>;
+export declare const getDiff: (storage: RepoStorage, latest: CID, earliest: CID | null) => Promise<Uint8Array>;
+export declare const getFullRepo: (storage: RepoStorage, cid: CID) => Promise<Uint8Array>;
+export declare const writeCommitsToCarStream: (storage: RepoStorage, car: BlockWriter, latest: CID, earliest: CID | null) => Promise<void>;
+export declare const getRecords: (storage: RepoStorage, commitCid: CID, paths: RecordPath[]) => Promise<Uint8Array>;