@clioplaylists/clio 0.1.5 → 0.1.7

package/dist/dataplane/server/db/pagination.d.ts

@@ -0,0 +1,86 @@
+import { AnyQb, DbRef } from './util';
+export type Cursor = {
+    primary: string;
+    secondary: string;
+};
+export type LabeledResult = {
+    primary: string | number;
+    secondary: string | number;
+};
+/**
+ * The GenericKeyset is an abstract class that sets-up the interface and partial implementation
+ * of a keyset-paginated cursor with two parts. There are three types involved:
+ *  - Result: a raw result (i.e. a row from the db) containing data that will make-up a cursor.
+ *    - E.g. { createdAt: '2022-01-01T12:00:00Z', cid: 'bafyx' }
+ *  - LabeledResult: a Result processed such that the "primary" and "secondary" parts of the cursor are labeled.
+ *    - E.g. { primary: '2022-01-01T12:00:00Z', secondary: 'bafyx' }
+ *  - Cursor: the two string parts that make-up the packed/string cursor.
+ *    - E.g. packed cursor '1641038400000__bafyx' in parts { primary: '1641038400000', secondary: 'bafyx' }
+ *
+ * These types relate as such. Implementers define the relations marked with a *:
+ *   Result -*-> LabeledResult <-*-> Cursor <--> packed/string cursor
+ *                     ↳ SQL Condition
+ */
+export declare abstract class GenericKeyset<R, LR extends LabeledResult> {
+    primary: DbRef;
+    secondary: DbRef;
+    constructor(primary: DbRef, secondary: DbRef);
+    abstract labelResult(result: R): LR;
+    abstract labeledResultToCursor(labeled: LR): Cursor;
+    abstract cursorToLabeledResult(cursor: Cursor): LR;
+    packFromResult(results: R | R[]): string | undefined;
+    pack(labeled?: LR): string | undefined;
+    unpack(cursorStr?: string): LR | undefined;
+    packCursor(cursor?: Cursor): string | undefined;
+    unpackCursor(cursorStr?: string): Cursor | undefined;
+    getSql(labeled?: LR, direction?: 'asc' | 'desc', tryIndex?: boolean): import("kysely").RawBuilder<unknown> | undefined;
+}
+type SortAtCidResult = {
+    sortAt: string;
+    cid: string;
+};
+type TimeCidLabeledResult = Cursor;
+export declare class TimeCidKeyset<TimeCidResult = SortAtCidResult> extends GenericKeyset<TimeCidResult, TimeCidLabeledResult> {
+    labelResult(result: TimeCidResult): TimeCidLabeledResult;
+    labeledResultToCursor(labeled: TimeCidLabeledResult): {
+        primary: string;
+        secondary: string;
+    };
+    cursorToLabeledResult(cursor: Cursor): {
+        primary: string;
+        secondary: string;
+    };
+}
+export declare class CreatedAtDidKeyset extends TimeCidKeyset<{
+    createdAt: string;
+    did: string;
+}> {
+    labelResult(result: {
+        createdAt: string;
+        did: string;
+    }): {
+        primary: string;
+        secondary: string;
+    };
+}
+export declare class IndexedAtDidKeyset extends TimeCidKeyset<{
+    indexedAt: string;
+    did: string;
+}> {
+    labelResult(result: {
+        indexedAt: string;
+        did: string;
+    }): {
+        primary: string;
+        secondary: string;
+    };
+}
+export declare const paginate: <QB extends AnyQb, K extends GenericKeyset<unknown, any>>(qb: QB, opts: {
+    limit?: number;
+    cursor?: string;
+    direction?: "asc" | "desc";
+    keyset: K;
+    tryIndex?: boolean;
+    nullsLast?: boolean;
+}) => QB;
+export {};

package/dist/dataplane/server/db/pagination.js

@@ -0,0 +1,128 @@
+import { InvalidRequestError } from '@atproto/xrpc-server';
+import { sql } from 'kysely';
+/**
+ * The GenericKeyset is an abstract class that sets-up the interface and partial implementation
+ * of a keyset-paginated cursor with two parts. There are three types involved:
+ *  - Result: a raw result (i.e. a row from the db) containing data that will make-up a cursor.
+ *    - E.g. { createdAt: '2022-01-01T12:00:00Z', cid: 'bafyx' }
+ *  - LabeledResult: a Result processed such that the "primary" and "secondary" parts of the cursor are labeled.
+ *    - E.g. { primary: '2022-01-01T12:00:00Z', secondary: 'bafyx' }
+ *  - Cursor: the two string parts that make-up the packed/string cursor.
+ *    - E.g. packed cursor '1641038400000__bafyx' in parts { primary: '1641038400000', secondary: 'bafyx' }
+ *
+ * These types relate as such. Implementers define the relations marked with a *:
+ *   Result -*-> LabeledResult <-*-> Cursor <--> packed/string cursor
+ *                     ↳ SQL Condition
+ */
+export class GenericKeyset {
+    primary;
+    secondary;
+    constructor(primary, secondary) {
+        this.primary = primary;
+        this.secondary = secondary;
+    }
+    packFromResult(results) {
+        const result = Array.isArray(results) ? results.at(-1) : results;
+        if (!result)
+            return;
+        return this.pack(this.labelResult(result));
+    }
+    pack(labeled) {
+        if (!labeled)
+            return;
+        const cursor = this.labeledResultToCursor(labeled);
+        return this.packCursor(cursor);
+    }
+    unpack(cursorStr) {
+        const cursor = this.unpackCursor(cursorStr);
+        if (!cursor)
+            return;
+        return this.cursorToLabeledResult(cursor);
+    }
+    packCursor(cursor) {
+        if (!cursor)
+            return;
+        return `${cursor.primary}__${cursor.secondary}`;
+    }
+    unpackCursor(cursorStr) {
+        if (!cursorStr)
+            return;
+        const result = cursorStr.split('__');
+        const [primary, secondary, ...others] = result;
+        if (!primary || !secondary || others.length > 0) {
+            throw new InvalidRequestError('Malformed cursor');
+        }
+        return {
+            primary,
+            secondary,
+        };
+    }
+    getSql(labeled, direction, tryIndex) {
+        if (labeled === undefined)
+            return;
+        if (tryIndex) {
+            // The tryIndex param will likely disappear and become the default implementation: here for now for gradual rollout query-by-query.
+            if (direction === 'asc') {
+                return sql `((${this.primary}, ${this.secondary}) > (${labeled.primary}, ${labeled.secondary}))`;
+            }
+            else {
+                return sql `((${this.primary}, ${this.secondary}) < (${labeled.primary}, ${labeled.secondary}))`;
+            }
+        }
+        else {
+            // @NOTE this implementation can struggle to use an index on (primary, secondary) for pagination due to the "or" usage.
+            if (direction === 'asc') {
+                return sql `((${this.primary} > ${labeled.primary}) or (${this.primary} = ${labeled.primary} and ${this.secondary} > ${labeled.secondary}))`;
+            }
+            else {
+                return sql `((${this.primary} < ${labeled.primary}) or (${this.primary} = ${labeled.primary} and ${this.secondary} < ${labeled.secondary}))`;
+            }
+        }
+    }
+}
+export class TimeCidKeyset extends GenericKeyset {
+    labelResult(result) {
+        return { primary: result.sortAt, secondary: result.cid };
+    }
+    labeledResultToCursor(labeled) {
+        return {
+            primary: new Date(labeled.primary).getTime().toString(),
+            secondary: labeled.secondary,
+        };
+    }
+    cursorToLabeledResult(cursor) {
+        const primaryDate = new Date(parseInt(cursor.primary, 10));
+        if (isNaN(primaryDate.getTime())) {
+            throw new InvalidRequestError('Malformed cursor');
+        }
+        return {
+            primary: primaryDate.toISOString(),
+            secondary: cursor.secondary,
+        };
+    }
+}
+export class CreatedAtDidKeyset extends TimeCidKeyset {
+    labelResult(result) {
+        return { primary: result.createdAt, secondary: result.did };
+    }
+}
+export class IndexedAtDidKeyset extends TimeCidKeyset {
+    labelResult(result) {
+        return { primary: result.indexedAt, secondary: result.did };
+    }
+}
+export const paginate = (qb, opts) => {
+    const { limit, cursor, keyset, direction = 'desc', tryIndex, nullsLast, } = opts;
+    const keysetSql = keyset.getSql(keyset.unpack(cursor), direction, tryIndex);
+    return qb
+        .if(!!limit, (q) => q.limit(limit))
+        .if(!nullsLast, (q) => q.orderBy(keyset.primary, direction).orderBy(keyset.secondary, direction))
+        .if(!!nullsLast, (q) => q
+        .orderBy(direction === 'asc'
+        ? sql `${keyset.primary} asc nulls last`
+        : sql `${keyset.primary} desc nulls last`)
+        .orderBy(direction === 'asc'
+        ? sql `${keyset.secondary} asc nulls last`
+        : sql `${keyset.secondary} desc nulls last`))
+        .if(!!keysetSql, (qb) => (keysetSql ? qb.where(keysetSql) : qb));
+};

package/dist/dataplane/server/db/tables/actor-sync.d.ts

@@ -0,0 +1,9 @@
+export declare const tableName = "actor_sync";
+export interface ActorSync {
+    did: string;
+    commitCid: string;
+    repoRev: string | null;
+}
+export type PartialDB = {
+    [tableName]: ActorSync;
+};

package/dist/dataplane/server/db/tables/actor-sync.js

@@ -0,0 +1 @@
+export const tableName = 'actor_sync';

package/dist/dataplane/server/db/tables/actor.d.ts

@@ -0,0 +1,11 @@
+export declare const tableName = "actor";
+export interface ActorTable {
+    did: string;
+    handle?: string | null;
+    indexed_at: string;
+    takedownRef?: string | null;
+    upstreamStatus?: string | null;
+}
+export type PartialDB = {
+    [tableName]: ActorTable;
+};

package/dist/dataplane/server/db/tables/actor.js

@@ -0,0 +1 @@
+export const tableName = 'actor';

package/dist/dataplane/server/db/tables/artist-list-item.d.ts

@@ -0,0 +1,11 @@
+import { Generated } from 'kysely';
+export declare const tableName = "artist_list_item";
+export interface ArtistListItemTable {
+    id: Generated<number>;
+    song_id: number;
+    seq_num: number;
+    artist_id: number;
+}
+export type PartialDB = {
+    [tableName]: ArtistListItemTable;
+};

package/dist/dataplane/server/db/tables/artist-list-item.js

@@ -0,0 +1 @@
+export const tableName = 'artist_list_item';

package/dist/dataplane/server/db/tables/artist.d.ts

@@ -0,0 +1,10 @@
+import { Generated } from 'kysely';
+export declare const tableName = "artist";
+export interface ArtistTable {
+    id: Generated<number>;
+    name: string;
+    mb_id: string;
+}
+export type PartialDB = {
+    [tableName]: ArtistTable;
+};

package/dist/dataplane/server/db/tables/artist.js

@@ -0,0 +1 @@
+export const tableName = 'artist';

package/dist/dataplane/server/db/tables/playlist-idea.d.ts

@@ -0,0 +1,14 @@
+export declare const tableName = "playlist_idea";
+export interface PlaylistIdeaTable {
+    uri: string;
+    title: string;
+    description?: string;
+    creator_did: string;
+    owner_did: string;
+    master_playlist_id?: number;
+    recommended_playlist_id?: number;
+    created_at: string;
+}
+export type PartialDB = {
+    [tableName]: PlaylistIdeaTable;
+};

package/dist/dataplane/server/db/tables/playlist-idea.js

@@ -0,0 +1 @@
+export const tableName = 'playlist_idea';

package/dist/dataplane/server/db/tables/playlist-item.d.ts

@@ -0,0 +1,11 @@
+import { Generated } from 'kysely';
+export declare const tableName = "playlist_item";
+export interface PlaylistItemTable {
+    id: Generated<number>;
+    playlist_id: number;
+    song_id: number;
+    position: number;
+}
+export type PartialDB = {
+    [tableName]: PlaylistItemTable;
+};

package/dist/dataplane/server/db/tables/playlist-item.js

@@ -0,0 +1 @@
+export const tableName = 'playlist_item';

package/dist/dataplane/server/db/tables/playlist.d.ts

@@ -0,0 +1,10 @@
+import { Generated } from 'kysely';
+export declare const tableName = "playlist";
+export interface PlaylistTable {
+    id: Generated<number>;
+    playlist_idea_uri: string;
+    type: string;
+}
+export type PartialDB = {
+    [tableName]: PlaylistTable;
+};

package/dist/dataplane/server/db/tables/playlist.js

@@ -0,0 +1 @@
+export const tableName = 'playlist';

package/dist/dataplane/server/db/tables/profile.d.ts

@@ -0,0 +1,15 @@
+export declare const tableName = "profile";
+export interface ProfileTable {
+    uri: string;
+    cid: string;
+    creator: string;
+    display_name?: string;
+    description?: string;
+    avatar_cid?: string;
+    banner_cid?: string;
+    created_at: string;
+    indexed_at: string;
+}
+export type PartialDB = {
+    [tableName]: ProfileTable;
+};

package/dist/dataplane/server/db/tables/profile.js

@@ -0,0 +1 @@
+export const tableName = 'profile';

package/dist/dataplane/server/db/tables/record.d.ts

@@ -0,0 +1,12 @@
+export declare const tableName = "record";
+export interface RecordTable {
+    uri: string;
+    cid: string;
+    did: string;
+    json: string;
+    indexed_at: string;
+    takedown_ref?: string;
+}
+export type PartialDB = {
+    [tableName]: RecordTable;
+};

package/dist/dataplane/server/db/tables/record.js

@@ -0,0 +1 @@
+export const tableName = 'record';

package/dist/dataplane/server/db/tables/song.d.ts

@@ -0,0 +1,12 @@
+import { Generated } from 'kysely';
+export declare const tableName = "song";
+export interface SongTable {
+    id: Generated<number>;
+    track_name: string;
+    track_mb_id: string;
+    album_artwork_ref?: string;
+    created_at?: string;
+}
+export type PartialDB = {
+    [tableName]: SongTable;
+};

package/dist/dataplane/server/db/tables/song.js

@@ -0,0 +1 @@
+export const tableName = 'song';

package/dist/dataplane/server/db/types.d.ts

@@ -0,0 +1,9 @@
+import { Pool as PgPool } from 'pg';
+export type PgOptions = {
+    url: string;
+    pool?: PgPool;
+    schema?: string;
+    poolSize?: number;
+    poolMaxUses?: number;
+    poolIdleTimeoutMs?: number;
+};

package/dist/dataplane/server/db/types.js

@@ -0,0 +1 @@
+export {};

package/dist/dataplane/server/db/util.d.ts

@@ -0,0 +1,20 @@
+import { DummyDriver, DynamicModule, ExpressionBuilder, RawBuilder, SelectQueryBuilder, SqliteAdapter, SqliteIntrospector, SqliteQueryCompiler } from 'kysely';
+import { DatabaseSchema, DatabaseSchemaType } from './database-schema';
+export declare const actorWhereClause: (actor: string) => RawBuilder<0 | 1>;
+export declare const notSoftDeletedClause: (alias: DbRef) => RawBuilder<unknown>;
+export declare const softDeleted: (actorOrRecord: {
+    takedownRef: string | null;
+}) => boolean;
+export declare const countAll: RawBuilder<number>;
+export declare const excluded: <T>(db: DatabaseSchema, col: any) => RawBuilder<T>;
+export declare const noMatch: RawBuilder<unknown>;
+export declare const valuesList: (vals: unknown[]) => RawBuilder<unknown>;
+export declare const dummyDialect: {
+    createAdapter(): SqliteAdapter;
+    createDriver(): DummyDriver;
+    createIntrospector(db: any): SqliteIntrospector;
+    createQueryCompiler(): SqliteQueryCompiler;
+};
+export type DbRef = RawBuilder | ReturnType<DynamicModule['ref']>;
+export type Subquery = ExpressionBuilder<DatabaseSchemaType, any>;
+export type AnyQb = SelectQueryBuilder<any, any, any>;

package/dist/dataplane/server/db/util.js

@@ -0,0 +1,40 @@
+import { DummyDriver, SqliteAdapter, SqliteIntrospector, SqliteQueryCompiler, sql, } from 'kysely';
+export const actorWhereClause = (actor) => {
+    if (actor.startsWith('did:')) {
+        return sql `"actor"."did" = ${actor}`;
+    }
+    else {
+        return sql `"actor"."handle" = ${actor}`;
+    }
+};
+// Applies to actor or record table
+export const notSoftDeletedClause = (alias) => {
+    return sql `${alias}."takedownRef" is null`;
+};
+export const softDeleted = (actorOrRecord) => {
+    return actorOrRecord.takedownRef !== null;
+};
+export const countAll = sql `count(*)`;
+// For use with doUpdateSet()
+export const excluded = (db, col) => {
+    return sql `${db.dynamic.ref(`excluded.${col}`)}`;
+};
+export const noMatch = sql `1 = 0`;
+// Can be useful for large where-in clauses, to get the db to use a hash lookup on the list
+export const valuesList = (vals) => {
+    return sql `(values (${sql.join(vals, sql `), (`)}))`;
+};
+export const dummyDialect = {
+    createAdapter() {
+        return new SqliteAdapter();
+    },
+    createDriver() {
+        return new DummyDriver();
+    },
+    createIntrospector(db) {
+        return new SqliteIntrospector(db);
+    },
+    createQueryCompiler() {
+        return new SqliteQueryCompiler();
+    },
+};

package/dist/dataplane/server/index.d.ts

@@ -0,0 +1,11 @@
+import { IdResolver } from '@atproto/identity';
+import http from 'node:http';
+import { Database } from './db';
+export { RepoSubscription } from './subscription';
+export declare class DataPlaneServer {
+    server: http.Server;
+    idResolver: IdResolver;
+    constructor(server: http.Server, idResolver: IdResolver);
+    static create(db: Database, port: number, plcUrl?: string): Promise<DataPlaneServer>;
+    destroy(): Promise<void>;
+}

package/dist/dataplane/server/index.js

@@ -0,0 +1,36 @@
+import { IdResolver, MemoryCache } from '@atproto/identity';
+import { expressConnectMiddleware } from '@connectrpc/connect-express';
+import express from 'express';
+import events from 'node:events';
+import createRoutes from './routes';
+export { RepoSubscription } from './subscription';
+export class DataPlaneServer {
+    server;
+    idResolver;
+    constructor(server, idResolver) {
+        this.server = server;
+        this.idResolver = idResolver;
+    }
+    static async create(db, port, plcUrl) {
+        const app = express();
+        const didCache = new MemoryCache();
+        const idResolver = new IdResolver({ plcUrl, didCache });
+        const routes = createRoutes(db, idResolver);
+        app.use(expressConnectMiddleware({ routes }));
+        const server = app.listen(port);
+        await events.once(server, 'listening');
+        return new DataPlaneServer(server, idResolver);
+    }
+    async destroy() {
+        return new Promise((resolve, reject) => {
+            this.server.close((err) => {
+                if (err) {
+                    reject(err);
+                }
+                else {
+                    resolve();
+                }
+            });
+        });
+    }
+}

package/dist/dataplane/server/indexing/index.d.ts

@@ -0,0 +1,32 @@
+import { IdResolver } from '@atproto/identity';
+import { WriteOpAction } from '@atproto/repo';
+import { AtUri } from '@atproto/syntax';
+import { CID } from 'multiformats/cid';
+import { Database } from '../db';
+import * as PlaylistIdea from './plugins/playlist-idea';
+import * as Profile from './plugins/profile';
+import { RecordProcessor } from './processor';
+export declare class IndexingService {
+    db: Database;
+    idResolver: IdResolver;
+    records: {
+        playlistIdea: PlaylistIdea.PluginType;
+        profile: Profile.PluginType;
+    };
+    constructor(db: Database, idResolver: IdResolver);
+    transact(txn: Database): IndexingService;
+    indexRecord(uri: AtUri, cid: CID, obj: unknown, action: WriteOpAction.Create | WriteOpAction.Update, timestamp: string): Promise<void>;
+    deleteRecord(uri: AtUri, cascading?: boolean): Promise<void>;
+    indexHandle(did: string, timestamp: string, force?: boolean): Promise<void>;
+    indexRepo(did: string, commit?: string): Promise<void>;
+    updateActorStatus(did: string, active: boolean, status?: string): Promise<void>;
+    setCommitLastSeen(did: string, commit: CID, rev: string): Promise<void>;
+    getCurrentRecords(did: string): Promise<Record<string, {
+        uri: AtUri;
+        cid: CID;
+    }>>;
+    findIndexerForCollection(collection: string): RecordProcessor<unknown, unknown> | undefined;
+    deleteActor(did: string): Promise<void>;
+    private getActorIsHosted;
+    unindexActor(did: string): Promise<void>;
+}