@clioplaylists/clio 0.1.7 → 0.1.9

package/dist/config.js

@@ -1,5 +1,11 @@
-import assert from 'node:assert';
-export class ServerConfig {
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ServerConfig = void 0;
+const node_assert_1 = __importDefault(require("node:assert"));
+class ServerConfig {
     cfg;
     assignedPort;
     constructor(cfg) {
@@ -8,23 +14,46 @@ export class ServerConfig {
     static readEnv(overrides) {
         const version = overrides?.version || process.env.CLIO_VERSION || undefined;
         const debugMode = overrides?.debugMode || process.env.NODE_ENV !== 'production';
+        const publicUrl = overrides?.publicUrl ?? process.env.CLIO_PUBLIC_URL;
         const serverDid = overrides?.serverDid || process.env.CLIO_SERVER_DID || 'did:example:test';
         const didPlcUrl = overrides?.didPlcUrl || process.env.DID_PLC_URL || 'http://localhost:2582';
-        // const handleResolverNameservers = process.env.BSKY_HANDLE_RESOLVER_NAMESERVERS
-        //   ? process.env.BSKY_HANDLE_RESOLVER_NAMESERVERS.split(',')
-        //   : []
+        const dataplaneUrls = overrides?.dataplaneUrls ?? envList(process.env.CLIO_DATAPLANE_URLS);
+        const handleResolverNameservers = process.env.BSKY_HANDLE_RESOLVER_NAMESERVERS
+            ? process.env.BSKY_HANDLE_RESOLVER_NAMESERVERS.split(',')
+            : [];
+        const modServiceDid = overrides?.modServiceDid || process.env.CLIO_MOD_SERVICE_DID || '';
+        const adminPasswords = overrides?.adminPasswords ?? envList(process.env.CLIO_ADMIN_PASSWORDS);
+        const dbPostgresUrl = overrides?.dbPostgresUrl ?? process.env.CLIO_DB_POSTGRES_URL;
+        const dbPostgresSchema = overrides?.dbPostgresSchema ?? process.env.CLIO_DB_POSTGRES_SCHEMA;
+        const oauthScope = overrides?.oauthScope ?? process.env.CLIO_OAUTH_SCOPE ?? 'atproto';
+        const oauthAllowHttp = overrides?.oauthAllowHttp ?? envBool(process.env.CLIO_OAUTH_ALLOW_HTTP);
+        const proxyPreferCompressed = overrides?.proxyPreferCompressed ?? envBool(process.env.CLIO_PROXY_PREFER_COMPRESSED);
+        const blobRateLimitBypassKey = overrides?.blobRateLimitBypassKey ?? process.env.CLIO_BLOB_RATELIMIT_BYPASS_KEY;
+        const blobRateLimitBypassHostname = overrides?.blobRateLimitBypassHostname ?? process.env.CLIO_BLOB_RATELIMIT_BYPASS_HOSTNAME;
         const envPort = overrides?.port || parseInt(process.env.CLIO_PORT || '', 10);
         const port = isNaN(envPort) ? 2635 : envPort;
         return new ServerConfig({
             version,
             debugMode,
             port,
+            publicUrl,
             serverDid,
             didPlcUrl,
+            handleResolverNameservers,
+            dataplaneUrls,
+            dbPostgresUrl,
+            dbPostgresSchema,
+            modServiceDid,
+            adminPasswords,
+            oauthScope,
+            oauthAllowHttp,
+            proxyPreferCompressed,
+            blobRateLimitBypassKey,
+            blobRateLimitBypassHostname,
         });
     }
     assignPort(port) {
-        assert(!this.cfg.port || this.cfg.port === port, 'Conflicting port in config');
+        (0, node_assert_1.default)(!this.cfg.port || this.cfg.port === port, 'Conflicting port in config');
         this.assignedPort = port;
     }
     get version() {
@@ -37,13 +66,64 @@ export class ServerConfig {
         return this.assignedPort || this.cfg.port;
     }
     get localUrl() {
-        assert(this.port, 'No port assigned');
+        (0, node_assert_1.default)(this.port, 'No port assigned');
         return `http://localhost:${this.port}`;
     }
+    get publicUrl() {
+        return this.cfg.publicUrl ?? this.localUrl;
+    }
     get serverDid() {
         return this.cfg.serverDid;
     }
     get didPlcUrl() {
         return this.cfg.didPlcUrl;
     }
+    get dataplaneUrls() {
+        return this.cfg.dataplaneUrls;
+    }
+    get dbPostgresUrl() {
+        return this.cfg.dbPostgresUrl;
+    }
+    get dbPostgresSchema() {
+        return this.cfg.dbPostgresSchema;
+    }
+    get modServiceDid() {
+        return this.cfg.modServiceDid;
+    }
+    get adminPasswords() {
+        return this.cfg.adminPasswords;
+    }
+    get oauthScope() {
+        return this.cfg.oauthScope ?? 'atproto';
+    }
+    get oauthAllowHttp() {
+        return !!this.cfg.oauthAllowHttp;
+    }
+    get handleResolverNameservers() {
+        return this.cfg.handleResolverNameservers;
+    }
+    get proxyPreferCompressed() {
+        return !!this.cfg.proxyPreferCompressed;
+    }
+    get blobRateLimitBypassKey() {
+        return this.cfg.blobRateLimitBypassKey;
+    }
+    get blobRateLimitBypassHostname() {
+        return this.cfg.blobRateLimitBypassHostname;
+    }
+}
+exports.ServerConfig = ServerConfig;
+function envList(str) {
+    if (str === undefined || str.length === 0)
+        return [];
+    return str.split(',');
+}
+function envBool(str) {
+    if (str === undefined)
+        return undefined;
+    if (str === 'true' || str === '1')
+        return true;
+    if (str === 'false' || str === '0')
+        return false;
+    return undefined;
 }

package/dist/context.d.ts

@@ -1,11 +1,25 @@
+import { Keypair } from '@atproto/crypto';
+import { IdResolver } from '@atproto/identity';
+import { Dispatcher } from 'undici';
+import { AuthVerifier } from './auth-verifier';
 import { DataPlaneClient } from './client';
 import { ServerConfig } from './config';
+import { HostList } from './dataplane';
 export default class AppContext {
     private opts;
     constructor(opts: {
         cfg: ServerConfig;
         dataplane: DataPlaneClient;
+        dataplaneHostList: HostList;
+        signingKey: Keypair;
+        idResolver: IdResolver;
+        authVerifier: AuthVerifier;
+        blobDispatcher?: Dispatcher;
     });
     get cfg(): ServerConfig;
     get dataplane(): DataPlaneClient;
+    get signingKey(): Keypair;
+    get authVerifier(): AuthVerifier;
+    get idResolver(): IdResolver;
+    get blobDispatcher(): Dispatcher;
 }

package/dist/context.js

@@ -1,4 +1,7 @@
-export default class AppContext {
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const undici_1 = require("undici");
+class AppContext {
     opts;
     constructor(opts) {
         this.opts = opts;
@@ -9,4 +12,17 @@ export default class AppContext {
     get dataplane() {
         return this.opts.dataplane;
     }
+    get signingKey() {
+        return this.opts.signingKey;
+    }
+    get authVerifier() {
+        return this.opts.authVerifier;
+    }
+    get idResolver() {
+        return this.opts.idResolver;
+    }
+    get blobDispatcher() {
+        return this.opts.blobDispatcher ?? new undici_1.Agent();
+    }
 }
+exports.default = AppContext;

package/dist/dataplane/client/hosts.d.ts

@@ -0,0 +1,21 @@
+/**
+ * Interface for a reactive list of hosts, i.e. for use with the dataplane client.
+ */
+export interface HostList {
+    get: () => Iterable<string>;
+    onUpdate(handler: HostListHandler): void;
+}
+type HostListHandler = (hosts: Iterable<string>) => void;
+/**
+ * Maintains a reactive HostList based on a simple setter.
+ */
+export declare class BasicHostList implements HostList {
+    private hosts;
+    private handlers;
+    constructor(hosts: Iterable<string>);
+    get(): Iterable<string>;
+    set(hosts: Iterable<string>): void;
+    private update;
+    onUpdate(handler: HostListHandler): void;
+}
+export {};

package/dist/dataplane/client/hosts.js

@@ -0,0 +1,29 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BasicHostList = void 0;
+/**
+ * Maintains a reactive HostList based on a simple setter.
+ */
+class BasicHostList {
+    hosts;
+    handlers = [];
+    constructor(hosts) {
+        this.hosts = hosts;
+    }
+    get() {
+        return this.hosts;
+    }
+    set(hosts) {
+        this.hosts = hosts;
+        this.update();
+    }
+    update() {
+        for (const handler of this.handlers) {
+            handler(this.hosts);
+        }
+    }
+    onUpdate(handler) {
+        this.handlers.push(handler);
+    }
+}
+exports.BasicHostList = BasicHostList;

package/dist/dataplane/client/index.d.ts

@@ -0,0 +1,13 @@
+import { Client, Code, ConnectError } from '@connectrpc/connect';
+import { ClioService } from '../../rpc/clio_connect';
+import { HostList } from './hosts';
+export * from './hosts';
+export * from './util';
+export type DataPlaneClient = Client<typeof ClioService>;
+type HttpVersion = '1.1' | '2';
+export declare const createDataPlaneClient: (hostList: HostList, opts: {
+    httpVersion?: HttpVersion;
+    rejectUnauthorized?: boolean;
+}) => DataPlaneClient;
+export { Code };
+export declare const isDataplaneError: (err: unknown, code?: Code) => err is ConnectError;

package/dist/dataplane/client/index.js

@@ -0,0 +1,120 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.isDataplaneError = exports.Code = exports.createDataPlaneClient = void 0;
+const connect_1 = require("@connectrpc/connect");
+Object.defineProperty(exports, "Code", { enumerable: true, get: function () { return connect_1.Code; } });
+const connect_node_1 = require("@connectrpc/connect-node");
+const node_assert_1 = __importDefault(require("node:assert"));
+const node_crypto_1 = require("node:crypto");
+const clio_connect_1 = require("../../rpc/clio_connect");
+__exportStar(require("./hosts"), exports);
+__exportStar(require("./util"), exports);
+const MAX_RETRIES = 3;
+const createDataPlaneClient = (hostList, opts) => {
+    const clients = new DataPlaneClients(hostList, opts);
+    return (0, connect_1.makeAnyClient)(clio_connect_1.ClioService, (method) => {
+        return async (...args) => {
+            let tries = 0;
+            let error;
+            let remainingClients = clients.get();
+            while (tries < MAX_RETRIES) {
+                const client = randomElement(remainingClients);
+                (0, node_assert_1.default)(client, 'no clients available');
+                try {
+                    return await client[method.localName](...args);
+                }
+                catch (err) {
+                    if (err instanceof connect_1.ConnectError && (err.code === connect_1.Code.Unavailable || err.code === connect_1.Code.Aborted)) {
+                        tries++;
+                        error = err;
+                        remainingClients = getRemainingClients(remainingClients, client);
+                    }
+                    else {
+                        throw err;
+                    }
+                }
+            }
+            (0, node_assert_1.default)(error);
+            throw error;
+        };
+    });
+};
+exports.createDataPlaneClient = createDataPlaneClient;
+/**
+ * Uses a reactive HostList in order to maintain a pool of DataPlaneClients.
+ * Each DataPlaneClient is cached per host so that it maintains connections
+ * and other internal state when the underlying HostList is updated.
+ */
+class DataPlaneClients {
+    hostList;
+    clientOpts;
+    clients = [];
+    clientsByHost = new Map();
+    constructor(hostList, clientOpts) {
+        this.hostList = hostList;
+        this.clientOpts = clientOpts;
+        this.refresh();
+        this.hostList.onUpdate(() => this.refresh());
+    }
+    get() {
+        return this.clients;
+    }
+    refresh() {
+        this.clients = [];
+        for (const host of this.hostList.get()) {
+            let client = this.clientsByHost.get(host);
+            if (!client) {
+                client = this.createClient(host);
+                this.clientsByHost.set(host, client);
+            }
+            this.clients.push(client);
+        }
+    }
+    createClient(host) {
+        return createBaseClient(host, this.clientOpts);
+    }
+}
+const isDataplaneError = (err, code) => {
+    if (err instanceof connect_1.ConnectError) {
+        return !code || err.code === code;
+    }
+    return false;
+};
+exports.isDataplaneError = isDataplaneError;
+const createBaseClient = (baseUrl, opts) => {
+    const { httpVersion = '2', rejectUnauthorized = true } = opts;
+    const transport = (0, connect_node_1.createGrpcTransport)({
+        baseUrl,
+        httpVersion,
+        acceptCompression: [],
+        nodeOptions: { rejectUnauthorized },
+    });
+    return (0, connect_1.createClient)(clio_connect_1.ClioService, transport);
+};
+const getRemainingClients = (clients, lastClient) => {
+    if (clients.length < 2)
+        return clients; // no clients to choose from
+    return clients.filter((c) => c !== lastClient);
+};
+const randomElement = (arr) => {
+    if (arr.length === 0)
+        return;
+    return arr[(0, node_crypto_1.randomInt)(arr.length)];
+};

package/dist/dataplane/client/util.d.ts

@@ -0,0 +1,20 @@
+import { Code, ConnectError } from '@connectrpc/connect';
+export declare const isDataplaneError: (err: unknown, code?: Code) => err is ConnectError;
+export declare const unpackIdentityServices: (servicesBytes: Uint8Array) => UnpackedServices;
+export declare const unpackIdentityKeys: (keysBytes: Uint8Array) => UnpackedKeys;
+export declare const getServiceEndpoint: (services: UnpackedServices, opts: {
+    id: string;
+    type: string;
+}) => string | undefined;
+export declare const getKeyAsDidKey: (keys: UnpackedKeys, opts: {
+    id: string;
+}) => string | undefined;
+type UnpackedServices = Record<string, {
+    Type: string;
+    URL: string;
+}>;
+type UnpackedKeys = Record<string, {
+    Type: string;
+    PublicKeyMultibase: string;
+}>;
+export {};

package/dist/dataplane/client/util.js

@@ -0,0 +1,92 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getKeyAsDidKey = exports.getServiceEndpoint = exports.unpackIdentityKeys = exports.unpackIdentityServices = exports.isDataplaneError = void 0;
+const identity_1 = require("@atproto/identity");
+const connect_1 = require("@connectrpc/connect");
+const ui8 = __importStar(require("uint8arrays"));
+const isDataplaneError = (err, code) => {
+    if (err instanceof connect_1.ConnectError) {
+        return !code || err.code === code;
+    }
+    return false;
+};
+exports.isDataplaneError = isDataplaneError;
+const unpackIdentityServices = (servicesBytes) => {
+    const servicesStr = ui8.toString(servicesBytes, 'utf8');
+    if (!servicesStr)
+        return {};
+    return JSON.parse(servicesStr);
+};
+exports.unpackIdentityServices = unpackIdentityServices;
+const unpackIdentityKeys = (keysBytes) => {
+    const keysStr = ui8.toString(keysBytes, 'utf8');
+    if (!keysStr)
+        return {};
+    return JSON.parse(keysStr);
+};
+exports.unpackIdentityKeys = unpackIdentityKeys;
+const getServiceEndpoint = (services, opts) => {
+    const endpoint = services[opts.id] && services[opts.id].Type === opts.type && validateUrl(services[opts.id].URL);
+    return endpoint || undefined;
+};
+exports.getServiceEndpoint = getServiceEndpoint;
+const getKeyAsDidKey = (keys, opts) => {
+    const key = keys[opts.id] &&
+        (0, identity_1.getDidKeyFromMultibase)({
+            type: keys[opts.id].Type,
+            publicKeyMultibase: keys[opts.id].PublicKeyMultibase,
+        });
+    return key || undefined;
+};
+exports.getKeyAsDidKey = getKeyAsDidKey;
+const validateUrl = (urlStr) => {
+    let url;
+    try {
+        url = new URL(urlStr);
+    }
+    catch {
+        return undefined;
+    }
+    if (!['http:', 'https:'].includes(url.protocol)) {
+        return undefined;
+    }
+    else if (!url.hostname) {
+        return undefined;
+    }
+    else {
+        return urlStr;
+    }
+};

package/dist/dataplane/index.js

@@ -1,2 +1,18 @@
-export * from './client';
-export * from './server';
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./client"), exports);
+__exportStar(require("./server"), exports);

package/dist/dataplane/server/background.d.ts

@@ -2,7 +2,7 @@ import PQueue from 'p-queue';
 import { Database } from './db';
 export declare class BackgroundQueue {
     db: Database;
-    queue: PQueue<import("p-queue/dist/priority-queue").default, import("p-queue").DefaultAddOptions>;
+    queue: PQueue<import("p-queue/dist/priority-queue").default, import("p-queue").QueueAddOptions>;
     destroyed: boolean;
     constructor(db: Database);
     add(task: Task): void;

package/dist/dataplane/server/background.js

@@ -1,9 +1,15 @@
-import PQueue from 'p-queue';
-import { dbLogger } from '../../logger';
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BackgroundQueue = void 0;
+const p_queue_1 = __importDefault(require("p-queue"));
+const logger_1 = require("../../logger");
 // A simple queue for in-process, out-of-band/backgrounded work
-export class BackgroundQueue {
+class BackgroundQueue {
     db;
-    queue = new PQueue();
+    queue = new p_queue_1.default();
     destroyed = false;
     constructor(db) {
         this.db = db;
@@ -15,7 +21,7 @@ export class BackgroundQueue {
         this.queue
             .add(() => task(this.db))
             .catch((err) => {
-            dbLogger.error(err, 'background queue task failed');
+            logger_1.dbLogger.error(err, 'background queue task failed');
         });
     }
     async processAll() {
@@ -28,3 +34,4 @@ export class BackgroundQueue {
         await this.queue.onIdle();
     }
 }
+exports.BackgroundQueue = BackgroundQueue;

package/dist/dataplane/server/db/database-schema.d.ts

@@ -3,11 +3,15 @@ import * as actorTable from './tables/actor';
 import * as actorSyncTable from './tables/actor-sync';
 import * as artistTable from './tables/artist';
 import * as artistListItemTable from './tables/artist-list-item';
+import * as oauthSessionTable from './tables/oauth-session';
+import * as oauthStateTable from './tables/oauth-state';
 import * as playlistTable from './tables/playlist';
 import * as playlistIdeaTable from './tables/playlist-idea';
 import * as playlistItemTable from './tables/playlist-item';
 import * as profileTable from './tables/profile';
 import * as recordTable from './tables/record';
 import * as songTable from './tables/song';
-export type DatabaseSchemaType = actorTable.PartialDB & actorSyncTable.PartialDB & artistTable.PartialDB & artistListItemTable.PartialDB & playlistTable.PartialDB & playlistIdeaTable.PartialDB & playlistItemTable.PartialDB & profileTable.PartialDB & recordTable.PartialDB & songTable.PartialDB;
+import * as songRecommendationTable from './tables/song-recommendation';
+import * as subscriptionCursorTable from './tables/subscription-cursor';
+export type DatabaseSchemaType = actorTable.PartialDB & actorSyncTable.PartialDB & artistTable.PartialDB & artistListItemTable.PartialDB & oauthSessionTable.PartialDB & oauthStateTable.PartialDB & playlistTable.PartialDB & playlistIdeaTable.PartialDB & playlistItemTable.PartialDB & profileTable.PartialDB & recordTable.PartialDB & songTable.PartialDB & songRecommendationTable.PartialDB & subscriptionCursorTable.PartialDB;
 export type DatabaseSchema = Kysely<DatabaseSchemaType>;

package/dist/dataplane/server/db/database-schema.js

@@ -1 +1,2 @@
-export {};
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });