@atproto/sync 0.1.0

package/CHANGELOG.md

@@ -0,0 +1,12 @@
+# @atproto/sync
+
+## 0.1.0
+
+### Minor Changes
+
+- [#2752](https://github.com/bluesky-social/atproto/pull/2752) [`b15dec2f4`](https://github.com/bluesky-social/atproto/commit/b15dec2f4feb25ac91b169c83ccff1adbb5a9442) Thanks [@dholms](https://github.com/dholms)! - Introduced initial sync package for consuming firehose (com.atproto.sync.subscribeRepos)
+
+### Patch Changes
+
+- Updated dependencies [[`b15dec2f4`](https://github.com/bluesky-social/atproto/commit/b15dec2f4feb25ac91b169c83ccff1adbb5a9442)]:
+  - @atproto/repo@0.5.0

package/LICENSE.txt

@@ -0,0 +1,7 @@
+Dual MIT/Apache-2.0 License
+
+Copyright (c) 2022-2024 Bluesky PBC, and Contributors
+
+Except as otherwise noted in individual files, this software is licensed under the MIT license (<http://opensource.org/licenses/MIT>), or the Apache License, Version 2.0 (<http://www.apache.org/licenses/LICENSE-2.0>).
+
+Downstream projects and end users may chose either license individually, or both together, at their discretion. The motivation for this dual-licensing is the additional software patent assurance provided by Apache 2.0.

package/README.md

@@ -0,0 +1,93 @@
+# @atproto/sync: atproto sync tools
+
+TypeScript library for syncing data from the [atproto](https://atproto.com) network. Currently only supports firehose (relay) subscriptions
+
+[![NPM](https://img.shields.io/npm/v/@atproto/sync)](https://www.npmjs.com/package/@atproto/sync)
+[![Github CI Status](https://github.com/bluesky-social/atproto/actions/workflows/repo.yaml/badge.svg)](https://github.com/bluesky-social/atproto/actions/workflows/repo.yaml)
+
+## Usage
+
+The firehose class will spin up a websocket connection to `com.atproto.sync.subscribeRepos` on a given repo host (by default the Relay run by Bluesky).
+
+Each event will be parsed, authenticated, and then passed on to the supplied `handleEvt` which can handle indexing.
+
+On `Commit` events, the firehose will verify signatures and repo proofs to ensure that the event is authentic. This can be disabled with the `unauthenticatedCommits` flag. Similarly on `Identity` events, the firehose will fetch the latest DID document for the repo and do bidirectional verification on the associated handle. This can be disabled with the `unauthenticatedHandles` flag.
+
+Events of a certain type can be excluded using the `excludeIdentity`/`excludeAccount`/`excludeCommit` flags. And repo writes can be filtered down to specific collections using `filterCollections`. By default, all events are parsed and passed through to the handler. Note that this filtered currently happens client-side, though it is likely we will introduce server-side methods for doing so in the future.
+
+Non-fatal errors that are encountered will be passed to the required `onError` handler. In most cases, these can just be logged.
+
+When using the firehose class, events are processed serially. Each event must be finished being handled before the next one is parsed and authenticated.
+
+```ts
+import { Firehose } from '@atproto/sync'
+import { IdResolver } from '@atproto/identity'
+
+const idResolver = new IdResolver()
+const firehose = new Firehose({
+  idResolver,
+  service: 'wss://bsky.network',
+  handleEvt: async (evt) => {
+    if (evt.event === 'identity') {
+      // ...
+    } else if (evt.event === 'account') {
+      // ...
+    } else if (evt.event === 'create') {
+      // ...
+    } else if (evt.event === 'update') {
+      // ...
+    } else if (evt.event === 'delete') {
+      // ...
+    }
+  },
+  onError: (err) => {
+    console.error(err)
+  },
+  filterCollections: ['com.myexample.app'],
+})
+firehose.start()
+
+// on service shutdown
+await firehose.destroy()
+```
+
+For more robust indexing pipelines, it's recommended to use the supplied `MemoryRunner` class. This provides an in-memory partitioned queue. As events from a given repo must be processed in order, this allows events to be processed concurrently while still processing events from any given repo serially.
+
+The `MemoryRunner` also tracks an internal cursor based on the last finished consecutive work. This ensures that no events are dropped, although it does mean that some events may occassionally be replayed (if the websocket drops and reconnects) and therefore it's recommended that any indexing logic is idempotent. An optional `setCursor` parameter may be supplied to the `MemoryRunner` which can be used to persistently store the most recently processed cursor.
+
+```ts
+import { Firehose, MemoryRunner } from '@atproto/sync'
+import { IdResolver } from '@atproto/identity'
+
+const idResolver = new IdResolver()
+const runner = new MemoryRunner({
+  setCursor: (cursor) => {
+    // persist cursor
+  },
+})
+const firehose = new Firehose({
+  idResolver,
+  runner,
+  service: 'wss://bsky.network',
+  handleEvt: async (evt) => {
+    // ...
+  },
+  onError: (err) => {
+    console.error(err)
+  },
+})
+firehose.start()
+
+// on service shutdown
+await firehose.destroy()
+await runner.destroy()
+```
+
+## License
+
+This project is dual-licensed under MIT and Apache 2.0 terms:
+
+- MIT license ([LICENSE-MIT.txt](https://github.com/bluesky-social/atproto/blob/main/LICENSE-MIT.txt) or http://opensource.org/licenses/MIT)
+- Apache License, Version 2.0, ([LICENSE-APACHE.txt](https://github.com/bluesky-social/atproto/blob/main/LICENSE-APACHE.txt) or http://www.apache.org/licenses/LICENSE-2.0)
+
+Downstream projects and end users may chose either license individually, or both together, at their discretion. The motivation for this dual-licensing is the additional software patent assurance provided by Apache 2.0.

package/dist/events.d.ts

@@ -0,0 +1,49 @@
+import { DidDocument } from '@atproto/identity';
+import type { RepoRecord } from '@atproto/lexicon';
+import { BlockMap } from '@atproto/repo';
+import { AtUri } from '@atproto/syntax';
+import type { CID } from 'multiformats/cid';
+export type Event = CommitEvt | IdentityEvt | AccountEvt;
+export type CommitMeta = {
+    seq: number;
+    time: string;
+    commit: CID;
+    blocks: BlockMap;
+    rev: string;
+    uri: AtUri;
+    did: string;
+    collection: string;
+    rkey: string;
+};
+export type CommitEvt = Create | Update | Delete;
+export type Create = CommitMeta & {
+    event: 'create';
+    record: RepoRecord;
+    cid: CID;
+};
+export type Update = CommitMeta & {
+    event: 'update';
+    record: RepoRecord;
+    cid: CID;
+};
+export type Delete = CommitMeta & {
+    event: 'delete';
+};
+export type IdentityEvt = {
+    seq: number;
+    time: string;
+    event: 'identity';
+    did: string;
+    handle?: string;
+    didDocument?: DidDocument;
+};
+export type AccountEvt = {
+    seq: number;
+    time: string;
+    event: 'account';
+    did: string;
+    active: boolean;
+    status?: AccountStatus;
+};
+export type AccountStatus = 'takendown' | 'suspended' | 'deleted' | 'deactivated';
+//# sourceMappingURL=events.d.ts.map

package/dist/events.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"events.d.ts","sourceRoot":"","sources":["../src/events.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAC/C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAA;AAClD,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAA;AACxC,OAAO,EAAE,KAAK,EAAE,MAAM,iBAAiB,CAAA;AACvC,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,kBAAkB,CAAA;AAE3C,MAAM,MAAM,KAAK,GAAG,SAAS,GAAG,WAAW,GAAG,UAAU,CAAA;AAExD,MAAM,MAAM,UAAU,GAAG;IACvB,GAAG,EAAE,MAAM,CAAA;IACX,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,GAAG,CAAA;IACX,MAAM,EAAE,QAAQ,CAAA;IAChB,GAAG,EAAE,MAAM,CAAA;IACX,GAAG,EAAE,KAAK,CAAA;IACV,GAAG,EAAE,MAAM,CAAA;IACX,UAAU,EAAE,MAAM,CAAA;IAClB,IAAI,EAAE,MAAM,CAAA;CACb,CAAA;AAED,MAAM,MAAM,SAAS,GAAG,MAAM,GAAG,MAAM,GAAG,MAAM,CAAA;AAEhD,MAAM,MAAM,MAAM,GAAG,UAAU,GAAG;IAChC,KAAK,EAAE,QAAQ,CAAA;IACf,MAAM,EAAE,UAAU,CAAA;IAClB,GAAG,EAAE,GAAG,CAAA;CACT,CAAA;AAED,MAAM,MAAM,MAAM,GAAG,UAAU,GAAG;IAChC,KAAK,EAAE,QAAQ,CAAA;IACf,MAAM,EAAE,UAAU,CAAA;IAClB,GAAG,EAAE,GAAG,CAAA;CACT,CAAA;AAED,MAAM,MAAM,MAAM,GAAG,UAAU,GAAG;IAChC,KAAK,EAAE,QAAQ,CAAA;CAChB,CAAA;AAED,MAAM,MAAM,WAAW,GAAG;IACxB,GAAG,EAAE,MAAM,CAAA;IACX,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,UAAU,CAAA;IACjB,GAAG,EAAE,MAAM,CAAA;IACX,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,WAAW,CAAC,EAAE,WAAW,CAAA;CAC1B,CAAA;AAED,MAAM,MAAM,UAAU,GAAG;IACvB,GAAG,EAAE,MAAM,CAAA;IACX,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,SAAS,CAAA;IAChB,GAAG,EAAE,MAAM,CAAA;IACX,MAAM,EAAE,OAAO,CAAA;IACf,MAAM,CAAC,EAAE,aAAa,CAAA;CACvB,CAAA;AAED,MAAM,MAAM,aAAa,GACrB,WAAW,GACX,WAAW,GACX,SAAS,GACT,aAAa,CAAA"}

package/dist/events.js

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=events.js.map

package/dist/events.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"events.js","sourceRoot":"","sources":["../src/events.ts"],"names":[],"mappings":""}

package/dist/firehose/index.d.ts

@@ -0,0 +1,50 @@
+import { IdResolver } from '@atproto/identity';
+import { type Account, type Commit, type Identity, type RepoEvent } from './lexicons';
+import { Event, CommitEvt, AccountEvt, IdentityEvt } from '../events';
+import { EventRunner } from '../runner';
+export type FirehoseOptions = {
+    idResolver: IdResolver;
+    handleEvent: (evt: Event) => Awaited<void>;
+    onError: (err: Error) => void;
+    getCursor?: () => Awaited<number | undefined>;
+    runner?: EventRunner;
+    service?: string;
+    subscriptionReconnectDelay?: number;
+    unauthenticatedCommits?: boolean;
+    unauthenticatedHandles?: boolean;
+    filterCollections?: string[];
+    excludeIdentity?: boolean;
+    excludeAccount?: boolean;
+    excludeCommit?: boolean;
+};
+export declare class Firehose {
+    opts: FirehoseOptions;
+    private sub;
+    private abortController;
+    private destoryDefer;
+    constructor(opts: FirehoseOptions);
+    start(): any;
+    private parseEvt;
+    private processEvt;
+    destroy(): Promise<void>;
+}
+export declare const parseCommitAuthenticated: (idResolver: IdResolver, evt: Commit, filterCollections?: string[], forceKeyRefresh?: boolean) => Promise<CommitEvt[]>;
+export declare const parseCommitUnauthenticated: (evt: Commit, filterCollections?: string[]) => Promise<CommitEvt[]>;
+export declare const parseIdentity: (idResolver: IdResolver, evt: Identity, unauthenticated?: boolean) => Promise<IdentityEvt | null>;
+export declare const parseAccount: (evt: Account) => AccountEvt | undefined;
+export declare class FirehoseValidationError extends Error {
+    value: unknown;
+    constructor(err: unknown, value: unknown);
+}
+export declare class FirehoseParseError extends Error {
+    event: RepoEvent;
+    constructor(err: unknown, event: RepoEvent);
+}
+export declare class FirehoseSubscriptionError extends Error {
+    constructor(err: unknown);
+}
+export declare class FirehoseHandlerError extends Error {
+    event: Event;
+    constructor(err: unknown, event: Event);
+}
+//# sourceMappingURL=index.d.ts.map

package/dist/firehose/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/firehose/index.ts"],"names":[],"mappings":"AACA,OAAO,EACL,UAAU,EAGX,MAAM,mBAAmB,CAAA;AAW1B,OAAO,EACL,KAAK,OAAO,EACZ,KAAK,MAAM,EACX,KAAK,QAAQ,EACb,KAAK,SAAS,EAMf,MAAM,YAAY,CAAA;AACnB,OAAO,EACL,KAAK,EAEL,SAAS,EACT,UAAU,EAEV,WAAW,EACZ,MAAM,WAAW,CAAA;AAElB,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAA;AAGvC,MAAM,MAAM,eAAe,GAAG;IAC5B,UAAU,EAAE,UAAU,CAAA;IAEtB,WAAW,EAAE,CAAC,GAAG,EAAE,KAAK,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;IAC1C,OAAO,EAAE,CAAC,GAAG,EAAE,KAAK,KAAK,IAAI,CAAA;IAC7B,SAAS,CAAC,EAAE,MAAM,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAAA;IAE7C,MAAM,CAAC,EAAE,WAAW,CAAA;IAEpB,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,0BAA0B,CAAC,EAAE,MAAM,CAAA;IAEnC,sBAAsB,CAAC,EAAE,OAAO,CAAA;IAChC,sBAAsB,CAAC,EAAE,OAAO,CAAA;IAEhC,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAA;IAC5B,eAAe,CAAC,EAAE,OAAO,CAAA;IACzB,cAAc,CAAC,EAAE,OAAO,CAAA;IACxB,aAAa,CAAC,EAAE,OAAO,CAAA;CACxB,CAAA;AAED,qBAAa,QAAQ;IAKA,IAAI,EAAE,eAAe;IAJxC,OAAO,CAAC,GAAG,CAAyB;IACpC,OAAO,CAAC,eAAe,CAAiB;IACxC,OAAO,CAAC,YAAY,CAAY;gBAEb,IAAI,EAAE,eAAe;IA6BlC,KAAK;YAiCG,QAAQ;YA6BR,UAAU;IAWlB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAI/B;AAED,eAAO,MAAM,wBAAwB,eACvB,UAAU,OACjB,MAAM,sBACS,MAAM,EAAE,gCAE3B,QAAQ,SAAS,EAAE,CAgCrB,CAAA;AAED,eAAO,MAAM,0BAA0B,QAChC,MAAM,sBACS,MAAM,EAAE,KAC3B,QAAQ,SAAS,EAAE,CAGrB,CAAA;AAyDD,eAAO,MAAM,aAAa,eACZ,UAAU,OACjB,QAAQ,gCAEZ,QAAQ,WAAW,GAAG,IAAI,CAe5B,CAAA;AAeD,eAAO,MAAM,YAAY,QAAS,OAAO,KAAG,UAAU,GAAG,SAUxD,CAAA;AAMD,qBAAa,uBAAwB,SAAQ,KAAK;IAGvC,KAAK,EAAE,OAAO;gBADrB,GAAG,EAAE,OAAO,EACL,KAAK,EAAE,OAAO;CAIxB;AAED,qBAAa,kBAAmB,SAAQ,KAAK;IAGlC,KAAK,EAAE,SAAS;gBADvB,GAAG,EAAE,OAAO,EACL,KAAK,EAAE,SAAS;CAI1B;AAED,qBAAa,yBAA0B,SAAQ,KAAK;gBACtC,GAAG,EAAE,OAAO;CAGzB;AAED,qBAAa,oBAAqB,SAAQ,KAAK;IAGpC,KAAK,EAAE,KAAK;gBADnB,GAAG,EAAE,OAAO,EACL,KAAK,EAAE,KAAK;CAItB"}

package/dist/firehose/index.js

@@ -0,0 +1,309 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.FirehoseHandlerError = exports.FirehoseSubscriptionError = exports.FirehoseParseError = exports.FirehoseValidationError = exports.parseAccount = exports.parseIdentity = exports.parseCommitUnauthenticated = exports.parseCommitAuthenticated = exports.Firehose = void 0;
+const common_1 = require("@atproto/common");
+const identity_1 = require("@atproto/identity");
+const repo_1 = require("@atproto/repo");
+const syntax_1 = require("@atproto/syntax");
+const xrpc_server_1 = require("@atproto/xrpc-server");
+const lexicons_1 = require("./lexicons");
+const util_1 = require("../util");
+class Firehose {
+    constructor(opts) {
+        Object.defineProperty(this, "opts", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: opts
+        });
+        Object.defineProperty(this, "sub", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "abortController", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "destoryDefer", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.destoryDefer = (0, common_1.createDeferrable)();
+        this.abortController = new AbortController();
+        if (this.opts.getCursor && this.opts.runner) {
+            throw new Error('Must set only `getCursor` or `runner`');
+        }
+        this.sub = new xrpc_server_1.Subscription({
+            service: opts.service ?? 'wss://bsky.network',
+            method: 'com.atproto.sync.subscribeRepos',
+            signal: this.abortController.signal,
+            getParams: async () => {
+                const getCursorFn = () => this.opts.runner?.getCursor() ?? this.opts.getCursor;
+                if (!getCursorFn) {
+                    return undefined;
+                }
+                const cursor = await getCursorFn();
+                return { cursor };
+            },
+            validate: (value) => {
+                try {
+                    return (0, lexicons_1.isValidRepoEvent)(value);
+                }
+                catch (err) {
+                    this.opts.onError(new FirehoseValidationError(err, value));
+                }
+            },
+        });
+    }
+    async start() {
+        try {
+            for await (const evt of this.sub) {
+                if (this.opts.runner) {
+                    const parsed = (0, util_1.didAndSeqForEvt)(evt);
+                    if (!parsed) {
+                        continue;
+                    }
+                    this.opts.runner.trackEvent(parsed.did, parsed.seq, async () => {
+                        const parsed = await this.parseEvt(evt);
+                        for (const write of parsed) {
+                            try {
+                                await this.opts.handleEvent(write);
+                            }
+                            catch (err) {
+                                this.opts.onError(new FirehoseHandlerError(err, write));
+                            }
+                        }
+                    });
+                }
+                else {
+                    await this.processEvt(evt);
+                }
+            }
+        }
+        catch (err) {
+            if (err && err['name'] === 'AbortError') {
+                this.destoryDefer.resolve();
+                return;
+            }
+            this.opts.onError(new FirehoseSubscriptionError(err));
+            await (0, common_1.wait)(this.opts.subscriptionReconnectDelay ?? 3000);
+            return this.start();
+        }
+    }
+    async parseEvt(evt) {
+        try {
+            if ((0, lexicons_1.isCommit)(evt) && !this.opts.excludeCommit) {
+                return this.opts.unauthenticatedCommits
+                    ? await (0, exports.parseCommitUnauthenticated)(evt, this.opts.filterCollections)
+                    : await (0, exports.parseCommitAuthenticated)(this.opts.idResolver, evt, this.opts.filterCollections);
+            }
+            else if ((0, lexicons_1.isAccount)(evt) && !this.opts.excludeAccount) {
+                const parsed = (0, exports.parseAccount)(evt);
+                return parsed ? [parsed] : [];
+            }
+            else if ((0, lexicons_1.isIdentity)(evt) && !this.opts.excludeIdentity) {
+                const parsed = await (0, exports.parseIdentity)(this.opts.idResolver, evt, this.opts.unauthenticatedHandles);
+                return parsed ? [parsed] : [];
+            }
+            else {
+                return [];
+            }
+        }
+        catch (err) {
+            this.opts.onError(new FirehoseParseError(err, evt));
+            return [];
+        }
+    }
+    async processEvt(evt) {
+        const parsed = await this.parseEvt(evt);
+        for (const write of parsed) {
+            try {
+                await this.opts.handleEvent(write);
+            }
+            catch (err) {
+                this.opts.onError(new FirehoseHandlerError(err, write));
+            }
+        }
+    }
+    async destroy() {
+        this.abortController.abort();
+        await this.destoryDefer.complete;
+    }
+}
+exports.Firehose = Firehose;
+const parseCommitAuthenticated = async (idResolver, evt, filterCollections, forceKeyRefresh = false) => {
+    const did = evt.repo;
+    const key = await idResolver.did.resolveAtprotoKey(did, forceKeyRefresh);
+    const claims = maybeFilterOps(evt.ops, filterCollections).map((op) => {
+        const { collection, rkey } = (0, repo_1.parseDataKey)(op.path);
+        return {
+            collection,
+            rkey,
+            cid: op.action === 'delete' ? null : op.cid,
+        };
+    });
+    const verifiedCids = {};
+    try {
+        const results = await (0, repo_1.verifyProofs)(evt.blocks, claims, did, key);
+        results.verified.forEach((op) => {
+            const path = (0, repo_1.formatDataKey)(op.collection, op.rkey);
+            verifiedCids[path] = op.cid;
+        });
+    }
+    catch (err) {
+        if (err instanceof repo_1.RepoVerificationError && !forceKeyRefresh) {
+            return (0, exports.parseCommitAuthenticated)(idResolver, evt, filterCollections, true);
+        }
+        throw err;
+    }
+    const verifiedOps = evt.ops.filter((op) => {
+        if (op.action === 'delete') {
+            return verifiedCids[op.path] === null;
+        }
+        else {
+            return op.cid !== null && op.cid.equals(verifiedCids[op.path]);
+        }
+    });
+    return formatCommitOps(evt, verifiedOps);
+};
+exports.parseCommitAuthenticated = parseCommitAuthenticated;
+const parseCommitUnauthenticated = async (evt, filterCollections) => {
+    const ops = maybeFilterOps(evt.ops, filterCollections);
+    return formatCommitOps(evt, ops);
+};
+exports.parseCommitUnauthenticated = parseCommitUnauthenticated;
+const maybeFilterOps = (ops, filterCollections) => {
+    if (!filterCollections)
+        return ops;
+    return ops.filter((op) => {
+        const { collection } = (0, repo_1.parseDataKey)(op.path);
+        return filterCollections.includes(collection);
+    });
+};
+const formatCommitOps = async (evt, ops) => {
+    const car = await (0, repo_1.readCar)(evt.blocks);
+    const evts = [];
+    for (const op of ops) {
+        const uri = syntax_1.AtUri.make(evt.repo, op.path);
+        const meta = {
+            seq: evt.seq,
+            time: evt.time,
+            commit: evt.commit,
+            blocks: car.blocks,
+            rev: evt.rev,
+            uri,
+            did: uri.host,
+            collection: uri.collection,
+            rkey: uri.rkey,
+        };
+        if (op.action === 'create' || op.action === 'update') {
+            if (!op.cid)
+                continue;
+            const recordBytes = car.blocks.get(op.cid);
+            if (!recordBytes)
+                continue;
+            const record = (0, repo_1.cborToLexRecord)(recordBytes);
+            evts.push({
+                ...meta,
+                event: op.action,
+                cid: op.cid,
+                record,
+            });
+        }
+        if (op.action === 'delete') {
+            evts.push({
+                ...meta,
+                event: 'delete',
+            });
+        }
+    }
+    return evts;
+};
+const parseIdentity = async (idResolver, evt, unauthenticated = false) => {
+    const res = await idResolver.did.resolve(evt.did);
+    const handle = res && !unauthenticated
+        ? await verifyHandle(idResolver, evt.did, res)
+        : undefined;
+    return {
+        event: 'identity',
+        seq: evt.seq,
+        time: evt.time,
+        did: evt.did,
+        handle,
+        didDocument: res ?? undefined,
+    };
+};
+exports.parseIdentity = parseIdentity;
+const verifyHandle = async (idResolver, did, didDoc) => {
+    const { handle } = (0, identity_1.parseToAtprotoDocument)(didDoc);
+    if (!handle) {
+        return undefined;
+    }
+    const res = await idResolver.handle.resolve(handle);
+    return res === did ? handle : undefined;
+};
+const parseAccount = (evt) => {
+    if (evt.status && !isValidStatus(evt.status))
+        return;
+    return {
+        event: 'account',
+        seq: evt.seq,
+        time: evt.time,
+        did: evt.did,
+        active: evt.active,
+        status: evt.status,
+    };
+};
+exports.parseAccount = parseAccount;
+const isValidStatus = (str) => {
+    return ['takendown', 'suspended', 'deleted', 'deactivated'].includes(str);
+};
+class FirehoseValidationError extends Error {
+    constructor(err, value) {
+        super('error in firehose event lexicon validation', { cause: err });
+        Object.defineProperty(this, "value", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: value
+        });
+    }
+}
+exports.FirehoseValidationError = FirehoseValidationError;
+class FirehoseParseError extends Error {
+    constructor(err, event) {
+        super('error in parsing and authenticating firehose event', { cause: err });
+        Object.defineProperty(this, "event", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: event
+        });
+    }
+}
+exports.FirehoseParseError = FirehoseParseError;
+class FirehoseSubscriptionError extends Error {
+    constructor(err) {
+        super('error on firehose subscription', { cause: err });
+    }
+}
+exports.FirehoseSubscriptionError = FirehoseSubscriptionError;
+class FirehoseHandlerError extends Error {
+    constructor(err, event) {
+        super('error in firehose event handler', { cause: err });
+        Object.defineProperty(this, "event", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: event
+        });
+    }
+}
+exports.FirehoseHandlerError = FirehoseHandlerError;
+//# sourceMappingURL=index.js.map

package/dist/firehose/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/firehose/index.ts"],"names":[],"mappings":";;;AAAA,4CAAoE;AACpE,gDAI0B;AAC1B,wCAOsB;AACtB,4CAAuC;AACvC,sDAAmD;AACnD,yCAUmB;AAWnB,kCAAyC;AAuBzC,MAAa,QAAQ;IAKnB,YAAmB,IAAqB;QAA5B;;;;mBAAO,IAAI;WAAiB;QAJhC;;;;;WAA4B;QAC5B;;;;;WAAgC;QAChC;;;;;WAAwB;QAG9B,IAAI,CAAC,YAAY,GAAG,IAAA,yBAAgB,GAAE,CAAA;QACtC,IAAI,CAAC,eAAe,GAAG,IAAI,eAAe,EAAE,CAAA;QAC5C,IAAI,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YAC5C,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAA;QAC1D,CAAC;QACD,IAAI,CAAC,GAAG,GAAG,IAAI,0BAAY,CAAC;YAC1B,OAAO,EAAE,IAAI,CAAC,OAAO,IAAI,oBAAoB;YAC7C,MAAM,EAAE,iCAAiC;YACzC,MAAM,EAAE,IAAI,CAAC,eAAe,CAAC,MAAM;YACnC,SAAS,EAAE,KAAK,IAAI,EAAE;gBACpB,MAAM,WAAW,GAAG,GAAG,EAAE,CACvB,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,SAAS,CAAA;gBACtD,IAAI,CAAC,WAAW,EAAE,CAAC;oBACjB,OAAO,SAAS,CAAA;gBAClB,CAAC;gBACD,MAAM,MAAM,GAAG,MAAM,WAAW,EAAE,CAAA;gBAClC,OAAO,EAAE,MAAM,EAAE,CAAA;YACnB,CAAC;YACD,QAAQ,EAAE,CAAC,KAAc,EAAE,EAAE;gBAC3B,IAAI,CAAC;oBACH,OAAO,IAAA,2BAAgB,EAAC,KAAK,CAAC,CAAA;gBAChC,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,uBAAuB,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAA;gBAC5D,CAAC;YACH,CAAC;SACF,CAAC,CAAA;IACJ,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,CAAC;YACH,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;gBACjC,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;oBACrB,MAAM,MAAM,GAAG,IAAA,sBAAe,EAAC,GAAG,CAAC,CAAA;oBACnC,IAAI,CAAC,MAAM,EAAE,CAAC;wBACZ,SAAQ;oBACV,CAAC;oBACD,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,KAAK,IAAI,EAAE;wBAC7D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAA;wBACvC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;4BAC3B,IAAI,CAAC;gCACH,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAA;4BACpC,CAAC;4BAAC,OAAO,GAAG,EAAE,CAAC;gCACb,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,oBAAoB,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAA;4BACzD,CAAC;wBACH,CAAC;oBACH,CAAC,CAAC,CAAA;gBACJ,CAAC;qBAAM,CAAC;oBACN,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAA;gBAC5B,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,KAAK,YAAY,EAAE,CAAC;gBACxC,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,CAAA;gBAC3B,OAAM;YACR,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,yBAAyB,CAAC,GAAG,CAAC,CAAC,CAAA;YACrD,MAAM,IAAA,aAAI,EAAC,IAAI,CAAC,IAAI,CAAC,0BAA0B,IAAI,IAAI,CAAC,CAAA;YACxD,OAAO,IAAI,CAAC,KAAK,EAAE,CAAA;QACrB,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,QAAQ,CAAC,GAAc;QACnC,IAAI,CAAC;YACH,IAAI,IAAA,mBAAQ,EAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;gBAC9C,OAAO,IAAI,CAAC,IAAI,CAAC,sBAAsB;oBACrC,CAAC,CAAC,MAAM,IAAA,kCAA0B,EAAC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC;oBACpE,CAAC,CAAC,MAAM,IAAA,gCAAwB,EAC5B,IAAI,CAAC,IAAI,CAAC,UAAU,EACpB,GAAG,EACH,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAC5B,CAAA;YACP,CAAC;iBAAM,IAAI,IAAA,oBAAS,EAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;gBACvD,MAAM,MAAM,GAAG,IAAA,oBAAY,EAAC,GAAG,CAAC,CAAA;gBAChC,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;YAC/B,CAAC;iBAAM,IAAI,IAAA,qBAAU,EAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;gBACzD,MAAM,MAAM,GAAG,MAAM,IAAA,qBAAa,EAChC,IAAI,CAAC,IAAI,CAAC,UAAU,EACpB,GAAG,EACH,IAAI,CAAC,IAAI,CAAC,sBAAsB,CACjC,CAAA;gBACD,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;YAC/B,CAAC;iBAAM,CAAC;gBACN,OAAO,EAAE,CAAA;YACX,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,kBAAkB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAA;YACnD,OAAO,EAAE,CAAA;QACX,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,UAAU,CAAC,GAAc;QACrC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAA;QACvC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAA;YACpC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,oBAAoB,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAA;YACzD,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,OAAO;QACX,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAA;QAC5B,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAA;IAClC,CAAC;CACF;AA/GD,4BA+GC;AAEM,MAAM,wBAAwB,GAAG,KAAK,EAC3C,UAAsB,EACtB,GAAW,EACX,iBAA4B,EAC5B,eAAe,GAAG,KAAK,EACD,EAAE;IACxB,MAAM,GAAG,GAAG,GAAG,CAAC,IAAI,CAAA;IACpB,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,iBAAiB,CAAC,GAAG,EAAE,eAAe,CAAC,CAAA;IACxE,MAAM,MAAM,GAAG,cAAc,CAAC,GAAG,CAAC,GAAG,EAAE,iBAAiB,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;QACnE,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,GAAG,IAAA,mBAAY,EAAC,EAAE,CAAC,IAAI,CAAC,CAAA;QAClD,OAAO;YACL,UAAU;YACV,IAAI;YACJ,GAAG,EAAE,EAAE,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG;SAC5C,CAAA;IACH,CAAC,CAAC,CAAA;IACF,MAAM,YAAY,GAA+B,EAAE,CAAA;IACnD,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,IAAA,mBAAY,EAAC,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,GAAG,CAAC,CAAA;QAChE,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE;YAC9B,MAAM,IAAI,GAAG,IAAA,oBAAa,EAAC,EAAE,CAAC,UAAU,EAAE,EAAE,CAAC,IAAI,CAAC,CAAA;YAClD,YAAY,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,GAAG,CAAA;QAC7B,CAAC,CAAC,CAAA;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,GAAG,YAAY,4BAAqB,IAAI,CAAC,eAAe,EAAE,CAAC;YAC7D,OAAO,IAAA,gCAAwB,EAAC,UAAU,EAAE,GAAG,EAAE,iBAAiB,EAAE,IAAI,CAAC,CAAA;QAC3E,CAAC;QACD,MAAM,GAAG,CAAA;IACX,CAAC;IACD,MAAM,WAAW,GAAa,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE;QAClD,IAAI,EAAE,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC3B,OAAO,YAAY,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,IAAI,CAAA;QACvC,CAAC;aAAM,CAAC;YACN,OAAO,EAAE,CAAC,GAAG,KAAK,IAAI,IAAI,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAA;QAChE,CAAC;IACH,CAAC,CAAC,CAAA;IACF,OAAO,eAAe,CAAC,GAAG,EAAE,WAAW,CAAC,CAAA;AAC1C,CAAC,CAAA;AArCY,QAAA,wBAAwB,4BAqCpC;AAEM,MAAM,0BAA0B,GAAG,KAAK,EAC7C,GAAW,EACX,iBAA4B,EACN,EAAE;IACxB,MAAM,GAAG,GAAG,cAAc,CAAC,GAAG,CAAC,GAAG,EAAE,iBAAiB,CAAC,CAAA;IACtD,OAAO,eAAe,CAAC,GAAG,EAAE,GAAG,CAAC,CAAA;AAClC,CAAC,CAAA;AANY,QAAA,0BAA0B,8BAMtC;AAED,MAAM,cAAc,GAAG,CACrB,GAAa,EACb,iBAA4B,EAClB,EAAE;IACZ,IAAI,CAAC,iBAAiB;QAAE,OAAO,GAAG,CAAA;IAClC,OAAO,GAAG,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE;QACvB,MAAM,EAAE,UAAU,EAAE,GAAG,IAAA,mBAAY,EAAC,EAAE,CAAC,IAAI,CAAC,CAAA;QAC5C,OAAO,iBAAiB,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAA;IAC/C,CAAC,CAAC,CAAA;AACJ,CAAC,CAAA;AAED,MAAM,eAAe,GAAG,KAAK,EAAE,GAAW,EAAE,GAAa,EAAE,EAAE;IAC3D,MAAM,GAAG,GAAG,MAAM,IAAA,cAAO,EAAC,GAAG,CAAC,MAAM,CAAC,CAAA;IAErC,MAAM,IAAI,GAAgB,EAAE,CAAA;IAE5B,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;QACrB,MAAM,GAAG,GAAG,cAAK,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,CAAA;QAEzC,MAAM,IAAI,GAAe;YACvB,GAAG,EAAE,GAAG,CAAC,GAAG;YACZ,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,GAAG,EAAE,GAAG,CAAC,GAAG;YACZ,GAAG;YACH,GAAG,EAAE,GAAG,CAAC,IAAI;YACb,UAAU,EAAE,GAAG,CAAC,UAAU;YAC1B,IAAI,EAAE,GAAG,CAAC,IAAI;SACf,CAAA;QAED,IAAI,EAAE,CAAC,MAAM,KAAK,QAAQ,IAAI,EAAE,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YACrD,IAAI,CAAC,EAAE,CAAC,GAAG;gBAAE,SAAQ;YACrB,MAAM,WAAW,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAA;YAC1C,IAAI,CAAC,WAAW;gBAAE,SAAQ;YAC1B,MAAM,MAAM,GAAG,IAAA,sBAAe,EAAC,WAAW,CAAC,CAAA;YAC3C,IAAI,CAAC,IAAI,CAAC;gBACR,GAAG,IAAI;gBACP,KAAK,EAAE,EAAE,CAAC,MAA6B;gBACvC,GAAG,EAAE,EAAE,CAAC,GAAG;gBACX,MAAM;aACP,CAAC,CAAA;QACJ,CAAC;QAED,IAAI,EAAE,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC3B,IAAI,CAAC,IAAI,CAAC;gBACR,GAAG,IAAI;gBACP,KAAK,EAAE,QAAQ;aAChB,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAA;AACb,CAAC,CAAA;AAEM,MAAM,aAAa,GAAG,KAAK,EAChC,UAAsB,EACtB,GAAa,EACb,eAAe,GAAG,KAAK,EACM,EAAE;IAC/B,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;IACjD,MAAM,MAAM,GACV,GAAG,IAAI,CAAC,eAAe;QACrB,CAAC,CAAC,MAAM,YAAY,CAAC,UAAU,EAAE,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC;QAC9C,CAAC,CAAC,SAAS,CAAA;IAEf,OAAO;QACL,KAAK,EAAE,UAAU;QACjB,GAAG,EAAE,GAAG,CAAC,GAAG;QACZ,IAAI,EAAE,GAAG,CAAC,IAAI;QACd,GAAG,EAAE,GAAG,CAAC,GAAG;QACZ,MAAM;QACN,WAAW,EAAE,GAAG,IAAI,SAAS;KAC9B,CAAA;AACH,CAAC,CAAA;AAnBY,QAAA,aAAa,iBAmBzB;AAED,MAAM,YAAY,GAAG,KAAK,EACxB,UAAsB,EACtB,GAAW,EACX,MAAmB,EACU,EAAE;IAC/B,MAAM,EAAE,MAAM,EAAE,GAAG,IAAA,iCAAsB,EAAC,MAAM,CAAC,CAAA;IACjD,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO,SAAS,CAAA;IAClB,CAAC;IACD,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;IACnD,OAAO,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAA;AACzC,CAAC,CAAA;AAEM,MAAM,YAAY,GAAG,CAAC,GAAY,EAA0B,EAAE;IACnE,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC;QAAE,OAAM;IACpD,OAAO;QACL,KAAK,EAAE,SAAS;QAChB,GAAG,EAAE,GAAG,CAAC,GAAG;QACZ,IAAI,EAAE,GAAG,CAAC,IAAI;QACd,GAAG,EAAE,GAAG,CAAC,GAAG;QACZ,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,MAAM,EAAE,GAAG,CAAC,MAAmC;KAChD,CAAA;AACH,CAAC,CAAA;AAVY,QAAA,YAAY,gBAUxB;AAED,MAAM,aAAa,GAAG,CAAC,GAAW,EAAwB,EAAE;IAC1D,OAAO,CAAC,WAAW,EAAE,WAAW,EAAE,SAAS,EAAE,aAAa,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAA;AAC3E,CAAC,CAAA;AAED,MAAa,uBAAwB,SAAQ,KAAK;IAChD,YACE,GAAY,EACL,KAAc;QAErB,KAAK,CAAC,4CAA4C,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAA;QAFnE;;;;mBAAO,KAAK;WAAS;IAGvB,CAAC;CACF;AAPD,0DAOC;AAED,MAAa,kBAAmB,SAAQ,KAAK;IAC3C,YACE,GAAY,EACL,KAAgB;QAEvB,KAAK,CAAC,oDAAoD,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAA;QAF3E;;;;mBAAO,KAAK;WAAW;IAGzB,CAAC;CACF;AAPD,gDAOC;AAED,MAAa,yBAA0B,SAAQ,KAAK;IAClD,YAAY,GAAY;QACtB,KAAK,CAAC,gCAAgC,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAA;IACzD,CAAC;CACF;AAJD,8DAIC;AAED,MAAa,oBAAqB,SAAQ,KAAK;IAC7C,YACE,GAAY,EACL,KAAY;QAEnB,KAAK,CAAC,iCAAiC,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAA;QAFxD;;;;mBAAO,KAAK;WAAO;IAGrB,CAAC;CACF;AAPD,oDAOC"}