@clioplaylists/clio 0.1.8 → 0.1.9

package/dist/dataplane/server/routes/records.js

@@ -1,15 +1,19 @@
-import { keyBy } from '@atproto/common';
-import { AtUri } from '@atproto/syntax';
-import { Timestamp } from '@bufbuild/protobuf';
-import * as ui8 from 'uint8arrays';
-import { ids } from '../../../lexicons/lexicons';
-import { Record } from '../../../rpc/clio_pb';
-export default (db) => ({
-    getProfileRecords: getRecords(db, ids.ComClioplaylistsAlphaActorProfile),
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getRecommendedPlaylistsForPlaylistIdea = exports.getRecords = void 0;
+const common_1 = require("@atproto/common");
+const syntax_1 = require("@atproto/syntax");
+const protobuf_1 = require("@bufbuild/protobuf");
+const node_buffer_1 = require("node:buffer");
+const lexicons_1 = require("../../../lexicons/lexicons");
+const clio_pb_1 = require("../../../rpc/clio_pb");
+exports.default = (db) => ({
+    getProfileRecords: (0, exports.getRecords)(db, lexicons_1.ids.ComClioplaylistsAlphaActorProfile),
+    getRecommendedPlaylistsForPlaylistIdea: (0, exports.getRecommendedPlaylistsForPlaylistIdea)(db),
 });
-export const getRecords = (db, collection) => async (req) => {
+const getRecords = (db, collection) => async (req) => {
     const validUris = collection
-        ? req.uris.filter((uri) => new AtUri(uri).collection === collection)
+        ? req.uris.filter((uri) => new syntax_1.AtUri(uri).collection === collection)
         : req.uris;
     const res = validUris.length
         ? await db.db
@@ -18,34 +22,56 @@ export const getRecords = (db, collection) => async (req) => {
             .where('uri', 'in', validUris)
             .execute()
         : [];
-    const byUri = keyBy(res, 'uri');
+    const byUri = (0, common_1.keyBy)(res, 'uri');
     const records = req.uris.map((uri) => {
         const row = byUri.get(uri);
         const json = row ? row.json : JSON.stringify(null);
         const createdAtRaw = new Date(JSON.parse(json)?.['createdAt']);
         const createdAt = !isNaN(createdAtRaw.getTime())
-            ? Timestamp.fromDate(createdAtRaw)
+            ? protobuf_1.Timestamp.fromDate(createdAtRaw)
             : undefined;
         const indexedAt = row?.indexed_at
-            ? Timestamp.fromDate(new Date(row?.indexed_at))
+            ? protobuf_1.Timestamp.fromDate(new Date(row?.indexed_at))
             : undefined;
-        const recordBytes = ui8.fromString(json, 'utf8');
-        return new Record({
+        const recordBytes = node_buffer_1.Buffer.from(json, 'utf8');
+        return new clio_pb_1.Record({
             record: recordBytes,
             cid: row?.cid,
             createdAt,
             indexedAt,
-            sortedAt: compositeTime(createdAt, indexedAt),
             takenDown: !!row?.takedown_ref,
             takedownRef: row?.takedown_ref ?? undefined,
         });
     });
     return { records };
 };
-const compositeTime = (ts1, ts2) => {
-    if (!ts1)
-        return ts2;
-    if (!ts2)
-        return ts1;
-    return ts1.toDate() < ts2.toDate() ? ts1 : ts2;
+exports.getRecords = getRecords;
+const getRecommendedPlaylistsForPlaylistIdea = (db) => async (req) => {
+    const res = await db.db
+        .selectFrom('record as r')
+        .innerJoin('song_recommendation as sr', 'sr.uri', 'r.uri')
+        .select(['r.uri', 'r.cid', 'r.json', 'r.indexed_at', 'r.takedown_ref'])
+        .where('sr.playlist_idea_uri', '=', req.playlistIdeaUri)
+        .execute();
+    const records = res.map((row) => {
+        const json = row.json;
+        const createdAtRaw = new Date(JSON.parse(json)?.['createdAt']);
+        const createdAt = !isNaN(createdAtRaw.getTime())
+            ? protobuf_1.Timestamp.fromDate(createdAtRaw)
+            : undefined;
+        const indexedAt = row.indexed_at
+            ? protobuf_1.Timestamp.fromDate(new Date(row.indexed_at))
+            : undefined;
+        const recordBytes = node_buffer_1.Buffer.from(json, 'utf8');
+        return new clio_pb_1.Record({
+            record: recordBytes,
+            cid: row.cid,
+            createdAt,
+            indexedAt,
+            takenDown: !!row.takedown_ref,
+            takedownRef: row.takedown_ref ?? undefined,
+        });
+    });
+    return { records };
 };
+exports.getRecommendedPlaylistsForPlaylistIdea = getRecommendedPlaylistsForPlaylistIdea;

package/dist/dataplane/server/routes/sync.js

@@ -1,12 +1,14 @@
-export default (db) => ({
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = (db) => ({
     async getLatestRev(req) {
         const res = await db.db
             .selectFrom('actor_sync')
             .where('did', '=', req.actorDid)
-            .select('repoRev')
+            .select('repo_rev')
             .executeTakeFirst();
         return {
-            rev: res?.repoRev ?? undefined,
+            rev: res?.repo_rev ?? undefined,
         };
     },
 });

package/dist/dataplane/server/storage/subscription-cursor.d.ts

@@ -0,0 +1,3 @@
+import { Database } from '../db';
+export declare const getSubscriptionCursor: (db: Database, service: string) => Promise<number>;
+export declare const setSubscriptionCursor: (db: Database, service: string, cursor: number) => Promise<void>;

package/dist/dataplane/server/storage/subscription-cursor.js

@@ -0,0 +1,25 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.setSubscriptionCursor = exports.getSubscriptionCursor = void 0;
+const getSubscriptionCursor = async (db, service) => {
+    const row = await db.db
+        .selectFrom('subscription_cursor')
+        .select(['cursor'])
+        .where('service', '=', service)
+        .executeTakeFirst();
+    return row?.cursor ?? 0;
+};
+exports.getSubscriptionCursor = getSubscriptionCursor;
+const setSubscriptionCursor = async (db, service, cursor) => {
+    const updatedAt = Date.now();
+    await db.db
+        .insertInto('subscription_cursor')
+        .values({
+        service,
+        cursor,
+        updated_at: updatedAt,
+    })
+        .onConflict((oc) => oc.column('service').doUpdateSet({ cursor, updated_at: updatedAt }))
+        .execute();
+};
+exports.setSubscriptionCursor = setSubscriptionCursor;

package/dist/dataplane/server/subscription.d.ts

@@ -9,8 +9,8 @@ export declare class RepoSubscription {
         db: Database;
         idResolver: IdResolver;
     };
-    firehose: Firehose;
-    runner: MemoryRunner;
+    firehose?: Firehose;
+    runner?: MemoryRunner;
     background: BackgroundQueue;
     indexingSvc: IndexingService;
     constructor(opts: {
@@ -18,8 +18,11 @@ export declare class RepoSubscription {
         db: Database;
         idResolver: IdResolver;
     });
-    start(): void;
+    start(): Promise<void>;
     restart(): Promise<void>;
     processAll(): Promise<void>;
     destroy(): Promise<void>;
+    private initPromise?;
+    private ensureInit;
+    private init;
 }

package/dist/dataplane/server/subscription.js

@@ -1,9 +1,13 @@
-import { WriteOpAction } from '@atproto/repo';
-import { Firehose, MemoryRunner } from '@atproto/sync';
-import { subLogger as log } from '../../logger';
-import { BackgroundQueue } from './background';
-import { IndexingService } from './indexing';
-export class RepoSubscription {
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RepoSubscription = void 0;
+const repo_1 = require("@atproto/repo");
+const sync_1 = require("@atproto/sync");
+const logger_1 = require("../../logger");
+const background_1 = require("./background");
+const indexing_1 = require("./indexing");
+const subscription_cursor_1 = require("./storage/subscription-cursor");
+class RepoSubscription {
     opts;
     firehose;
     runner;
@@ -11,76 +15,82 @@ export class RepoSubscription {
     indexingSvc;
     constructor(opts) {
         this.opts = opts;
-        const { service, db, idResolver } = opts;
-        this.background = new BackgroundQueue(db);
-        this.indexingSvc = new IndexingService(db, idResolver);
-        const { runner, firehose } = createFirehose({
-            idResolver,
-            service,
-            indexingSvc: this.indexingSvc,
-        });
-        this.runner = runner;
-        this.firehose = firehose;
+        const { db, idResolver } = opts;
+        this.background = new background_1.BackgroundQueue(db);
+        this.indexingSvc = new indexing_1.IndexingService(db, idResolver);
     }
-    start() {
-        this.firehose.start();
+    async start() {
+        await this.ensureInit();
+        this.firehose?.start();
     }
     async restart() {
         await this.destroy();
-        const { runner, firehose } = createFirehose({
-            idResolver: this.opts.idResolver,
-            service: this.opts.service,
-            indexingSvc: this.indexingSvc,
-        });
-        this.runner = runner;
-        this.firehose = firehose;
-        this.start();
+        await this.ensureInit(true);
+        await this.start();
     }
     async processAll() {
-        await this.runner.processAll();
+        await this.ensureInit();
+        await this.runner?.processAll();
         await this.background.processAll();
     }
     async destroy() {
-        await this.firehose.destroy();
-        await this.runner.destroy();
+        await this.firehose?.destroy();
+        await this.runner?.destroy();
         await this.background.processAll();
+        this.firehose = undefined;
+        this.runner = undefined;
+    }
+    initPromise;
+    async ensureInit(force = false) {
+        if (force)
+            this.initPromise = undefined;
+        if (!this.initPromise) {
+            this.initPromise = this.init();
+        }
+        await this.initPromise;
+    }
+    async init() {
+        const { idResolver, service, db } = this.opts;
+        const startCursor = await (0, subscription_cursor_1.getSubscriptionCursor)(db, service);
+        const runner = new sync_1.MemoryRunner({
+            startCursor,
+            setCursor: async (cursor) => {
+                await (0, subscription_cursor_1.setSubscriptionCursor)(db, service, cursor);
+            },
+        });
+        const firehose = new sync_1.Firehose({
+            idResolver,
+            runner,
+            service,
+            unauthenticatedHandles: true,
+            unauthenticatedCommits: true,
+            onError: (err) => logger_1.subLogger.error({ err }, 'error in subscription'),
+            handleEvent: createHandleEvent(this.indexingSvc),
+        });
+        this.runner = runner;
+        this.firehose = firehose;
     }
 }
-const createFirehose = (opts) => {
-    const { idResolver, service, indexingSvc } = opts;
-    const runner = new MemoryRunner({ startCursor: 0 });
-    const firehose = new Firehose({
-        idResolver,
-        runner,
-        service,
-        unauthenticatedHandles: true, // indexing service handles these
-        unauthenticatedCommits: true, // @TODO there seems to be a very rare issue where the authenticator thinks a block is missing in deletion ops
-        onError: (err) => log.error({ err }, 'error in subscription'),
-        handleEvent: async (evt) => {
-            if (evt.event === 'identity') {
-                await indexingSvc.indexHandle(evt.did, evt.time, true);
-            }
-            else if (evt.event === 'account') {
-                if (evt.active === false && evt.status === 'deleted') {
-                    await indexingSvc.deleteActor(evt.did);
-                }
-                else {
-                    await indexingSvc.updateActorStatus(evt.did, evt.active, evt.status);
-                }
+exports.RepoSubscription = RepoSubscription;
+const createHandleEvent = (indexingSvc) => {
+    return async (evt) => {
+        if (evt.event === 'identity') {
+            await indexingSvc.indexHandle(evt.did, evt.time, true);
+            return;
+        }
+        if (evt.event === 'account') {
+            const status = evt.status ?? '';
+            if (evt.active === false && status === 'deleted') {
+                await indexingSvc.deleteActor(evt.did);
             }
             else {
-                const indexFn = evt.event === 'delete'
-                    ? indexingSvc.deleteRecord(evt.uri)
-                    : indexingSvc.indexRecord(evt.uri, evt.cid, evt.record, evt.event === 'create'
-                        ? WriteOpAction.Create
-                        : WriteOpAction.Update, evt.time);
-                await Promise.all([
-                    indexFn,
-                    indexingSvc.setCommitLastSeen(evt.did, evt.commit, evt.rev),
-                    indexingSvc.indexHandle(evt.did, evt.time),
-                ]);
+                await indexingSvc.updateActorStatus(evt.did, evt.active, status);
             }
-        },
-    });
-    return { firehose, runner };
+            return;
+        }
+        const indexFn = evt.event === 'delete'
+            ? indexingSvc.deleteRecord(evt.uri)
+            : indexingSvc.indexRecord(evt.uri, evt.cid, evt.record, evt.event === 'create' ? repo_1.WriteOpAction.Create : repo_1.WriteOpAction.Update, evt.time);
+        await Promise.all([indexFn, indexingSvc.setCommitLastSeen(evt.did, evt.commit, evt.rev), indexingSvc.indexHandle(evt.did, evt.time)]);
+    };
 };

package/dist/error.js

@@ -1,10 +1,14 @@
-import { XRPCError } from '@atproto/xrpc-server';
-import { httpLogger as log } from './logger';
-export const handler = (err, _req, res, next) => {
-    log.error(err, 'unexpected internal server error');
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.handler = void 0;
+const xrpc_server_1 = require("@atproto/xrpc-server");
+const logger_1 = require("./logger");
+const handler = (err, _req, res, next) => {
+    logger_1.httpLogger.error(err, 'unexpected internal server error');
     if (res.headersSent) {
         return next(err);
     }
-    const serverError = XRPCError.fromError(err);
+    const serverError = xrpc_server_1.XRPCError.fromError(err);
     res.status(serverError.type).json(serverError.payload);
 };
+exports.handler = handler;

package/dist/index.d.ts

@@ -1,16 +1,19 @@
 import { Keypair } from '@atproto/crypto';
-import express from 'express';
 import { Server } from 'http';
 import { ServerConfig } from './config';
 import AppContext from './context';
+type ExpressApp = {
+    use: (...args: unknown[]) => unknown;
+    listen(port: number, callback?: () => void): Server;
+};
 export declare class ClioAppView {
     ctx: AppContext;
-    app: express.Application;
+    app: ExpressApp;
     server?: Server;
     private terminator?;
     constructor(opts: {
         ctx: AppContext;
-        app: express.Application;
+        app: ExpressApp;
     });
     static create(opts: {
         config: ServerConfig;

package/dist/index.js

@@ -1,18 +1,60 @@
-import { DAY, SECOND } from '@atproto/common';
-import cors from 'cors';
-import { config } from 'dotenv';
-import events from 'events';
-import express from 'express';
-import { createHttpTerminator } from 'http-terminator';
-import API, { health } from './api';
-import AppContext from './context';
-import { createDataPlaneClient } from './dataplane';
-import { BasicHostList } from './dataplane/client/hosts';
-import * as error from './error';
-import { createServer } from './lexicons';
-import { loggerMiddleware } from './logger';
-config();
-export class ClioAppView {
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ClioAppView = void 0;
+const common_1 = require("@atproto/common");
+const identity_1 = require("@atproto/identity");
+const cors_1 = __importDefault(require("cors"));
+const dotenv_1 = require("dotenv");
+const events_1 = __importDefault(require("events"));
+const express_1 = __importDefault(require("express"));
+const http_terminator_1 = require("http-terminator");
+const api_1 = __importStar(require("./api"));
+const oauth = __importStar(require("./api/oauth"));
+const auth_verifier_1 = require("./auth-verifier");
+const context_1 = __importDefault(require("./context"));
+const dataplane_1 = require("./dataplane");
+const hosts_1 = require("./dataplane/client/hosts");
+const error = __importStar(require("./error"));
+const lexicons_1 = require("./lexicons");
+const logger_1 = require("./logger");
+(0, dotenv_1.config)();
+class ClioAppView {
     ctx;
     app;
     server;
@@ -23,33 +65,53 @@ export class ClioAppView {
     }
     static create(opts) {
         const { config, signingKey } = opts;
-        const app = express();
-        app.use(cors({ maxAge: DAY / SECOND }));
-        app.use(loggerMiddleware);
-        const dataplaneHostList = new BasicHostList(config.dataplaneUrls);
-        const dataplane = createDataPlaneClient(dataplaneHostList, {
+        const app = (0, express_1.default)();
+        // used solely for handle resolution: identity lookups occur on dataplane
+        const idResolver = new identity_1.IdResolver({
+            plcUrl: config.didPlcUrl,
+            backupNameservers: config.handleResolverNameservers,
+        });
+        const dataplaneHostList = new hosts_1.BasicHostList(config.dataplaneUrls);
+        const dataplane = (0, dataplane_1.createDataPlaneClient)(dataplaneHostList, {
             httpVersion: '2',
             rejectUnauthorized: false,
         });
-        const server = createServer();
-        const ctx = new AppContext({
+        const authVerifier = new auth_verifier_1.AuthVerifier(dataplane, {
+            ownDid: config.serverDid,
+            alternateAudienceDids: [],
+            modServiceDid: '',
+            adminPasses: config.adminPasswords,
+        });
+        const server = (0, lexicons_1.createServer)();
+        const ctx = new context_1.default({
             cfg: config,
             dataplane,
             signingKey,
+            dataplaneHostList,
+            idResolver,
+            authVerifier,
         });
-        const api = API(server, ctx);
-        app.use(health.createRouter(ctx));
+        const api = (0, api_1.default)(server, ctx);
+        app.use(api_1.health.createRouter(ctx));
+        app.use(api_1.wellKnown.createRouter(ctx));
+        app.use(oauth.createRouter(ctx));
         app.use(api.xrpc.router);
         app.use(error.handler);
+        app.use((0, cors_1.default)({ maxAge: common_1.DAY / common_1.SECOND }));
+        app.use(logger_1.loggerMiddleware);
         return new ClioAppView({ ctx, app });
     }
     async start() {
         console.log('Starting Clio AppView...');
-        const server = this.app.listen(this.ctx.cfg.port);
+        const listenPort = this.ctx.cfg.port;
+        if (!listenPort) {
+            throw new Error('Missing port in config');
+        }
+        const server = this.app.listen(listenPort);
         this.server = server;
         server.keepAliveTimeout = 90000;
-        this.terminator = createHttpTerminator({ server });
-        await events.once(server, 'listening');
+        this.terminator = (0, http_terminator_1.createHttpTerminator)({ server });
+        await events_1.default.once(server, 'listening');
         const { port } = server.address();
         this.ctx.cfg.assignPort(port);
         return server;
@@ -58,4 +120,5 @@ export class ClioAppView {
         await this.terminator?.terminate();
     }
 }
-export default ClioAppView;
+exports.ClioAppView = ClioAppView;
+exports.default = ClioAppView;