@clioplaylists/clio 0.1.4 → 0.1.6

package/dist/dataplane/server/indexing/processor.d.ts

@@ -0,0 +1,22 @@
+import { AtUri } from '@atproto/syntax';
+import { CID } from 'multiformats/cid';
+import { Database } from '../db';
+import { DatabaseSchema } from '../db/database-schema';
+type RecordProcessorParams<T, S> = {
+    lexId: string;
+    insertFn: (db: DatabaseSchema, uri: AtUri, cid: CID, obj: T, timestamp: string) => Promise<S | null>;
+    findDuplicate: (db: DatabaseSchema, uri: AtUri, obj: T) => Promise<AtUri | null>;
+    deleteFn: (db: DatabaseSchema, uri: AtUri) => Promise<S | null>;
+};
+export declare class RecordProcessor<T, S> {
+    private params;
+    collection: string;
+    db: DatabaseSchema;
+    constructor(appDb: Database, params: RecordProcessorParams<T, S>);
+    matchesSchema(obj: unknown): obj is T;
+    assertValidRecord(obj: unknown): asserts obj is T;
+    insertRecord(uri: AtUri, cid: CID, obj: unknown, timestamp: string): Promise<void>;
+    updateRecord(uri: AtUri, cid: CID, obj: unknown, timestamp: string): Promise<void>;
+    deleteRecord(uri: AtUri, cascading?: boolean): Promise<void>;
+}
+export default RecordProcessor;

package/dist/dataplane/server/indexing/processor.js

@@ -0,0 +1,90 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RecordProcessor = void 0;
+const lexicon_1 = require("@atproto/lexicon");
+const lexicons_1 = require("../../../lexicons/lexicons");
+class RecordProcessor {
+    constructor(appDb, params) {
+        Object.defineProperty(this, "params", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: params
+        });
+        Object.defineProperty(this, "collection", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "db", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.db = appDb.db;
+        this.collection = this.params.lexId;
+    }
+    matchesSchema(obj) {
+        try {
+            this.assertValidRecord(obj);
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+    assertValidRecord(obj) {
+        lexicons_1.lexicons.assertValidRecord(this.params.lexId, obj);
+    }
+    async insertRecord(uri, cid, obj, timestamp) {
+        this.assertValidRecord(obj);
+        await this.db
+            .insertInto('record')
+            .values({
+            uri: uri.toString(),
+            cid: cid.toString(),
+            did: uri.host,
+            json: (0, lexicon_1.stringifyLex)(obj),
+            indexed_at: timestamp,
+        })
+            .onConflict((oc) => oc.doNothing())
+            .execute();
+        await this.params.insertFn(this.db, uri, cid, obj, timestamp);
+    }
+    // Currently using a very simple strategy for updates: purge the existing index
+    // for the uri then replace it. The main upside is that this allows the indexer
+    // for each collection to avoid bespoke logic for in-place updates, which isn't
+    // straightforward in the general case. We still get nice control over notifications.
+    async updateRecord(uri, cid, obj, timestamp) {
+        this.assertValidRecord(obj);
+        await this.db
+            .updateTable('record')
+            .where('uri', '=', uri.toString())
+            .set({
+            cid: cid.toString(),
+            json: (0, lexicon_1.stringifyLex)(obj),
+            indexed_at: timestamp,
+        })
+            .execute();
+        const deleted = await this.params.deleteFn(this.db, uri);
+        if (!deleted) {
+            // If a record was updated but hadn't been indexed yet, treat it like a plain insert.
+            return this.insertRecord(uri, cid, obj, timestamp);
+        }
+        const inserted = await this.params.insertFn(this.db, uri, cid, obj, timestamp);
+        if (!inserted) {
+            throw new Error('Record update failed: removed from index but could not be replaced');
+        }
+    }
+    async deleteRecord(uri, cascading = false) {
+        await this.db
+            .deleteFrom('record')
+            .where('uri', '=', uri.toString())
+            .execute();
+        await this.params.deleteFn(this.db, uri);
+    }
+}
+exports.RecordProcessor = RecordProcessor;
+exports.default = RecordProcessor;

package/dist/dataplane/server/routes/identity.d.ts

@@ -0,0 +1,6 @@
+import { IdResolver } from '@atproto/identity';
+import { ServiceImpl } from '@connectrpc/connect';
+import { ClioService } from '../../../rpc/clio_connect';
+import { Database } from '../db';
+declare const _default: (_db: Database, idResolver: IdResolver) => Partial<ServiceImpl<typeof ClioService>>;
+export default _default;

package/dist/dataplane/server/routes/identity.js

@@ -0,0 +1,56 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const identity_1 = require("@atproto/identity");
+const protobuf_1 = require("@bufbuild/protobuf");
+const connect_1 = require("@connectrpc/connect");
+exports.default = (_db, idResolver) => ({
+    async getIdentityByDid(req) {
+        const doc = await idResolver.did.resolve(req.did);
+        if (!doc) {
+            throw new connect_1.ConnectError('identity not found', connect_1.Code.NotFound);
+        }
+        return getResultFromDoc(doc);
+    },
+    async getIdentityByHandle(req) {
+        const did = await idResolver.handle.resolve(req.handle);
+        if (!did) {
+            throw new connect_1.ConnectError('identity not found', connect_1.Code.NotFound);
+        }
+        const doc = await idResolver.did.resolve(did);
+        if (!doc || did !== (0, identity_1.getDid)(doc)) {
+            throw new connect_1.ConnectError('identity not found', connect_1.Code.NotFound);
+        }
+        return getResultFromDoc(doc);
+    },
+});
+const getResultFromDoc = (doc) => {
+    const keys = {};
+    doc.verificationMethod?.forEach((method) => {
+        const id = method.id.split('#').at(1);
+        if (!id)
+            return;
+        keys[id] = {
+            Type: method.type,
+            PublicKeyMultibase: method.publicKeyMultibase || '',
+        };
+    });
+    const services = {};
+    doc.service?.forEach((service) => {
+        const id = service.id.split('#').at(1);
+        if (!id)
+            return;
+        if (typeof service.serviceEndpoint !== 'string')
+            return;
+        services[id] = {
+            Type: service.type,
+            URL: service.serviceEndpoint,
+        };
+    });
+    return {
+        did: (0, identity_1.getDid)(doc),
+        handle: (0, identity_1.getHandle)(doc),
+        keys: Buffer.from(JSON.stringify(keys)),
+        services: Buffer.from(JSON.stringify(services)),
+        updated: protobuf_1.Timestamp.fromDate(new Date()),
+    };
+};

package/dist/dataplane/server/routes/index.d.ts

@@ -0,0 +1,5 @@
+import { IdResolver } from '@atproto/identity';
+import { ConnectRouter } from '@connectrpc/connect';
+import { Database } from '../db';
+declare const _default: (db: Database, idResolver: IdResolver) => (router: ConnectRouter) => ConnectRouter;
+export default _default;

package/dist/dataplane/server/routes/index.js

@@ -0,0 +1,19 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const clio_connect_1 = require("../../../rpc/clio_connect");
+const identity_1 = __importDefault(require("./identity"));
+const profile_1 = __importDefault(require("./profile"));
+const records_1 = __importDefault(require("./records"));
+const sync_1 = __importDefault(require("./sync"));
+exports.default = (db, idResolver) => (router) => router.service(clio_connect_1.ClioService, {
+    ...(0, identity_1.default)(db, idResolver),
+    ...(0, profile_1.default)(db),
+    ...(0, records_1.default)(db),
+    ...(0, sync_1.default)(db),
+    async healthCheck() {
+        return {};
+    },
+});

package/dist/dataplane/server/routes/profile.d.ts

@@ -0,0 +1,5 @@
+import { ServiceImpl } from '@connectrpc/connect';
+import { ClioService } from '../../../rpc/clio_connect';
+import { Database } from '../db';
+declare const _default: (db: Database) => Partial<ServiceImpl<typeof ClioService>>;
+export default _default;

package/dist/dataplane/server/routes/profile.js

@@ -0,0 +1,61 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const common_1 = require("@atproto/common");
+const util_1 = require("../../../util");
+const records_1 = require("./records");
+exports.default = (db) => ({
+    async getActors(req) {
+        const { dids } = req;
+        if (dids.length === 0) {
+            return { actors: [] };
+        }
+        const profileUris = dids.map((did) => `at://${did}/app.bsky.actor.profile/self`);
+        const chatDeclarationUris = dids.map((did) => `at://${did}/chat.bsky.actor.declaration/self`);
+        const [handlesRes, profiles, chatDeclarations] = await Promise.all([
+            db.db.selectFrom('actor').selectAll().execute(),
+            (0, records_1.getRecords)(db)({ uris: profileUris }),
+            (0, records_1.getRecords)(db)({ uris: chatDeclarationUris }),
+        ]);
+        const byDid = (0, common_1.keyBy)(handlesRes, 'did');
+        const actors = dids.map((did, i) => {
+            const row = byDid.get(did);
+            const chatDeclaration = (0, util_1.parseRecordBytes)(chatDeclarations.records[i].record);
+            return {
+                exists: !!row,
+                handle: row?.handle ?? undefined,
+                profile: profiles.records[i],
+                takenDown: !!row?.takedownRef,
+                takedownRef: row?.takedownRef || undefined,
+                tombstonedAt: undefined, // in current implementation, tombstoned actors are deleted
+                allowIncomingChatsFrom: typeof chatDeclaration?.['allowIncoming'] === 'string'
+                    ? chatDeclaration['allowIncoming']
+                    : undefined,
+                upstreamStatus: row?.upstreamStatus ?? '',
+                createdAt: profiles.records[i].createdAt, // @NOTE profile creation date not trusted in production
+            };
+        });
+        return { actors };
+    },
+    async getDidsByHandles(req) {
+        const { handles } = req;
+        if (handles.length === 0) {
+            return { dids: [] };
+        }
+        const res = await db.db
+            .selectFrom('actor')
+            .where('handle', 'in', handles)
+            .selectAll()
+            .execute();
+        const byHandle = (0, common_1.keyBy)(res, 'handle');
+        const dids = handles.map((handle) => byHandle.get(handle)?.did ?? '');
+        return { dids };
+    },
+    async updateActorUpstreamStatus(req) {
+        const { actorDid, upstreamStatus } = req;
+        await db.db
+            .updateTable('actor')
+            .set({ upstreamStatus })
+            .where('did', '=', actorDid)
+            .execute();
+    },
+});

package/dist/dataplane/server/routes/records.d.ts

@@ -0,0 +1,11 @@
+import { ServiceImpl } from '@connectrpc/connect';
+import { ClioService } from '../../../rpc/clio_connect';
+import { Record } from '../../../rpc/clio_pb';
+import { Database } from '../db';
+declare const _default: (db: Database) => Partial<ServiceImpl<typeof ClioService>>;
+export default _default;
+export declare const getRecords: (db: Database, collection?: string) => (req: {
+    uris: string[];
+}) => Promise<{
+    records: Record[];
+}>;

package/dist/dataplane/server/routes/records.js

@@ -0,0 +1,88 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getRecords = void 0;
+const common_1 = require("@atproto/common");
+const syntax_1 = require("@atproto/syntax");
+const protobuf_1 = require("@bufbuild/protobuf");
+const ui8 = __importStar(require("uint8arrays"));
+const lexicons_1 = require("../../../lexicons/lexicons");
+const clio_pb_1 = require("../../../rpc/clio_pb");
+exports.default = (db) => ({
+    getProfileRecords: (0, exports.getRecords)(db, lexicons_1.ids.ComClioplaylistsAlphaActorProfile),
+});
+const getRecords = (db, collection) => async (req) => {
+    const validUris = collection
+        ? req.uris.filter((uri) => new syntax_1.AtUri(uri).collection === collection)
+        : req.uris;
+    const res = validUris.length
+        ? await db.db
+            .selectFrom('record')
+            .selectAll()
+            .where('uri', 'in', validUris)
+            .execute()
+        : [];
+    const byUri = (0, common_1.keyBy)(res, 'uri');
+    const records = req.uris.map((uri) => {
+        const row = byUri.get(uri);
+        const json = row ? row.json : JSON.stringify(null);
+        const createdAtRaw = new Date(JSON.parse(json)?.['createdAt']);
+        const createdAt = !isNaN(createdAtRaw.getTime())
+            ? protobuf_1.Timestamp.fromDate(createdAtRaw)
+            : undefined;
+        const indexedAt = row?.indexed_at
+            ? protobuf_1.Timestamp.fromDate(new Date(row?.indexed_at))
+            : undefined;
+        const recordBytes = ui8.fromString(json, 'utf8');
+        return new clio_pb_1.Record({
+            record: recordBytes,
+            cid: row?.cid,
+            createdAt,
+            indexedAt,
+            sortedAt: compositeTime(createdAt, indexedAt),
+            takenDown: !!row?.takedown_ref,
+            takedownRef: row?.takedown_ref ?? undefined,
+        });
+    });
+    return { records };
+};
+exports.getRecords = getRecords;
+const compositeTime = (ts1, ts2) => {
+    if (!ts1)
+        return ts2;
+    if (!ts2)
+        return ts1;
+    return ts1.toDate() < ts2.toDate() ? ts1 : ts2;
+};

package/dist/dataplane/server/routes/sync.d.ts

@@ -0,0 +1,5 @@
+import { ServiceImpl } from '@connectrpc/connect';
+import { ClioService } from '../../../rpc/clio_connect';
+import { Database } from '../db';
+declare const _default: (db: Database) => Partial<ServiceImpl<typeof ClioService>>;
+export default _default;

package/dist/dataplane/server/routes/sync.js

@@ -0,0 +1,14 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = (db) => ({
+    async getLatestRev(req) {
+        const res = await db.db
+            .selectFrom('actor_sync')
+            .where('did', '=', req.actorDid)
+            .select('repoRev')
+            .executeTakeFirst();
+        return {
+            rev: res?.repoRev ?? undefined,
+        };
+    },
+});

package/dist/dataplane/server/subscription.d.ts

@@ -0,0 +1,25 @@
+import { IdResolver } from '@atproto/identity';
+import { Firehose, MemoryRunner } from '@atproto/sync';
+import { BackgroundQueue } from './background';
+import { Database } from './db';
+import { IndexingService } from './indexing';
+export declare class RepoSubscription {
+    opts: {
+        service: string;
+        db: Database;
+        idResolver: IdResolver;
+    };
+    firehose: Firehose;
+    runner: MemoryRunner;
+    background: BackgroundQueue;
+    indexingSvc: IndexingService;
+    constructor(opts: {
+        service: string;
+        db: Database;
+        idResolver: IdResolver;
+    });
+    start(): void;
+    restart(): Promise<void>;
+    processAll(): Promise<void>;
+    destroy(): Promise<void>;
+}

package/dist/dataplane/server/subscription.js

@@ -0,0 +1,114 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RepoSubscription = void 0;
+const repo_1 = require("@atproto/repo");
+const sync_1 = require("@atproto/sync");
+const logger_1 = require("../../logger");
+const background_1 = require("./background");
+const indexing_1 = require("./indexing");
+class RepoSubscription {
+    constructor(opts) {
+        Object.defineProperty(this, "opts", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: opts
+        });
+        Object.defineProperty(this, "firehose", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "runner", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "background", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "indexingSvc", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        const { service, db, idResolver } = opts;
+        this.background = new background_1.BackgroundQueue(db);
+        this.indexingSvc = new indexing_1.IndexingService(db, idResolver);
+        const { runner, firehose } = createFirehose({
+            idResolver,
+            service,
+            indexingSvc: this.indexingSvc,
+        });
+        this.runner = runner;
+        this.firehose = firehose;
+    }
+    start() {
+        this.firehose.start();
+    }
+    async restart() {
+        await this.destroy();
+        const { runner, firehose } = createFirehose({
+            idResolver: this.opts.idResolver,
+            service: this.opts.service,
+            indexingSvc: this.indexingSvc,
+        });
+        this.runner = runner;
+        this.firehose = firehose;
+        this.start();
+    }
+    async processAll() {
+        await this.runner.processAll();
+        await this.background.processAll();
+    }
+    async destroy() {
+        await this.firehose.destroy();
+        await this.runner.destroy();
+        await this.background.processAll();
+    }
+}
+exports.RepoSubscription = RepoSubscription;
+const createFirehose = (opts) => {
+    const { idResolver, service, indexingSvc } = opts;
+    const runner = new sync_1.MemoryRunner({ startCursor: 0 });
+    const firehose = new sync_1.Firehose({
+        idResolver,
+        runner,
+        service,
+        unauthenticatedHandles: true, // indexing service handles these
+        unauthenticatedCommits: true, // @TODO there seems to be a very rare issue where the authenticator thinks a block is missing in deletion ops
+        onError: (err) => logger_1.subLogger.error({ err }, 'error in subscription'),
+        handleEvent: async (evt) => {
+            if (evt.event === 'identity') {
+                await indexingSvc.indexHandle(evt.did, evt.time, true);
+            }
+            else if (evt.event === 'account') {
+                if (evt.active === false && evt.status === 'deleted') {
+                    await indexingSvc.deleteActor(evt.did);
+                }
+                else {
+                    await indexingSvc.updateActorStatus(evt.did, evt.active, evt.status);
+                }
+            }
+            else {
+                const indexFn = evt.event === 'delete'
+                    ? indexingSvc.deleteRecord(evt.uri)
+                    : indexingSvc.indexRecord(evt.uri, evt.cid, evt.record, evt.event === 'create'
+                        ? repo_1.WriteOpAction.Create
+                        : repo_1.WriteOpAction.Update, evt.time);
+                await Promise.all([
+                    indexFn,
+                    indexingSvc.setCommitLastSeen(evt.did, evt.commit, evt.rev),
+                    indexingSvc.indexHandle(evt.did, evt.time),
+                ]);
+            }
+        },
+    });
+    return { firehose, runner };
+};

package/dist/error.d.ts

@@ -0,0 +1,2 @@
+import { ErrorRequestHandler } from 'express';
+export declare const handler: ErrorRequestHandler;

package/dist/error.js

@@ -0,0 +1,14 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.handler = void 0;
+const xrpc_server_1 = require("@atproto/xrpc-server");
+const logger_1 = require("./logger");
+const handler = (err, _req, res, next) => {
+    logger_1.httpLogger.error(err, 'unexpected internal server error');
+    if (res.headersSent) {
+        return next(err);
+    }
+    const serverError = xrpc_server_1.XRPCError.fromError(err);
+    res.status(serverError.type).json(serverError.payload);
+};
+exports.handler = handler;

package/dist/index.d.ts

@@ -0,0 +1,20 @@
+import express from 'express';
+import { Server } from 'http';
+import { ServerConfig } from './config';
+import AppContext from './context';
+export declare class ClioAppView {
+    ctx: AppContext;
+    app: express.Application;
+    server?: Server;
+    private terminator?;
+    constructor(opts: {
+        ctx: AppContext;
+        app: express.Application;
+    });
+    static create(opts: {
+        config: ServerConfig;
+    }): ClioAppView;
+    start(): Promise<Server>;
+    destroy(): Promise<void>;
+}
+export default ClioAppView;