@clioplaylists/clio 0.1.5 → 0.1.7

package/dist/dataplane/server/indexing/index.js

@@ -0,0 +1,271 @@
+import { Agent, ComAtprotoSyncGetLatestCommit } from '@atproto/api';
+import { DAY, HOUR } from '@atproto/common';
+import { getPds } from '@atproto/identity';
+import { ValidationError } from '@atproto/lexicon';
+import { WriteOpAction, getAndParseRecord, readCarWithRoot, verifyRepo, } from '@atproto/repo';
+import { AtUri } from '@atproto/syntax';
+import { sql } from 'kysely';
+import { CID } from 'multiformats/cid';
+import { httpLogger } from '../../../logger';
+import { retryXrpc } from '../../../util/retry';
+import * as PlaylistIdea from './plugins/playlist-idea';
+import * as Profile from './plugins/profile';
+export class IndexingService {
+    db;
+    idResolver;
+    records;
+    constructor(db, idResolver) {
+        this.db = db;
+        this.idResolver = idResolver;
+        this.records = {
+            playlistIdea: PlaylistIdea.makePlugin(this.db),
+            profile: Profile.makePlugin(this.db),
+        };
+    }
+    transact(txn) {
+        txn.assertTransaction();
+        return new IndexingService(txn, this.idResolver);
+    }
+    async indexRecord(uri, cid, obj, action, timestamp) {
+        this.db.assertNotTransaction();
+        await this.db.transaction(async (txn) => {
+            const indexingTx = this.transact(txn);
+            const indexer = indexingTx.findIndexerForCollection(uri.collection);
+            if (!indexer)
+                return;
+            if (action === WriteOpAction.Create) {
+                await indexer.insertRecord(uri, cid, obj, timestamp);
+            }
+            else {
+                await indexer.updateRecord(uri, cid, obj, timestamp);
+            }
+        });
+    }
+    async deleteRecord(uri, cascading = false) {
+        this.db.assertNotTransaction();
+        await this.db.transaction(async (txn) => {
+            const indexingTx = this.transact(txn);
+            const indexer = indexingTx.findIndexerForCollection(uri.collection);
+            if (!indexer)
+                return;
+            await indexer.deleteRecord(uri, cascading);
+        });
+    }
+    async indexHandle(did, timestamp, force = false) {
+        this.db.assertNotTransaction();
+        const actor = await this.db.db
+            .selectFrom('actor')
+            .where('did', '=', did)
+            .selectAll()
+            .executeTakeFirst();
+        if (!force && !needsHandleReindex(actor, timestamp)) {
+            return;
+        }
+        const atpData = await this.idResolver.did.resolveAtprotoData(did, true);
+        const handleToDid = await this.idResolver.handle.resolve(atpData.handle);
+        const handle = did === handleToDid ? atpData.handle.toLowerCase() : null;
+        const actorWithHandle = handle !== null
+            ? await this.db.db
+                .selectFrom('actor')
+                .where('handle', '=', handle)
+                .selectAll()
+                .executeTakeFirst()
+            : null;
+        // handle contention
+        if (handle && actorWithHandle && did !== actorWithHandle.did) {
+            await this.db.db
+                .updateTable('actor')
+                .where('actor.did', '=', actorWithHandle.did)
+                .set({ handle: null })
+                .execute();
+        }
+        const actorInfo = { handle, indexed_at: timestamp };
+        await this.db.db
+            .insertInto('actor')
+            .values({ did, ...actorInfo })
+            .onConflict((oc) => oc.column('did').doUpdateSet(actorInfo))
+            .returning('did')
+            .executeTakeFirst();
+    }
+    async indexRepo(did, commit) {
+        this.db.assertNotTransaction();
+        const now = new Date().toISOString();
+        const { pds, signingKey } = await this.idResolver.did.resolveAtprotoData(did, true);
+        const agent = new Agent(pds);
+        const { data: car } = await retryXrpc(() => agent.com.atproto.sync.getRepo({ did }));
+        const { root, blocks } = await readCarWithRoot(car);
+        const verifiedRepo = await verifyRepo(blocks, root, did, signingKey);
+        const currRecords = await this.getCurrentRecords(did);
+        const repoRecords = formatCheckout(did, verifiedRepo);
+        const diff = findDiffFromCheckout(currRecords, repoRecords);
+        await Promise.all(diff.map(async (op) => {
+            const { uri, cid } = op;
+            try {
+                if (op.op === 'delete') {
+                    await this.deleteRecord(uri);
+                }
+                else {
+                    const parsed = await getAndParseRecord(blocks, cid);
+                    await this.indexRecord(uri, cid, parsed.record, op.op === 'create' ? WriteOpAction.Create : WriteOpAction.Update, now);
+                }
+            }
+            catch (err) {
+                if (err instanceof ValidationError) {
+                    httpLogger.warn({ did, commit, uri: uri.toString(), cid: cid.toString() }, 'skipping indexing of invalid record');
+                }
+                else {
+                    httpLogger.error({ err, did, commit, uri: uri.toString(), cid: cid.toString() }, 'skipping indexing due to error processing record');
+                }
+            }
+        }));
+    }
+    async updateActorStatus(did, active, status = '') {
+        let upstreamStatus;
+        if (active) {
+            upstreamStatus = null;
+        }
+        else if (['deactivated', 'suspended', 'takendown'].includes(status)) {
+            upstreamStatus = status;
+        }
+        else {
+            throw new Error(`Unrecognized account status: ${status}`);
+        }
+        await this.db.db
+            .updateTable('actor')
+            .set({ upstreamStatus })
+            .where('did', '=', did)
+            .execute();
+    }
+    async setCommitLastSeen(did, commit, rev) {
+        const { ref } = this.db.db.dynamic;
+        await this.db.db
+            .insertInto('actor_sync')
+            .values({
+            did,
+            commitCid: commit.toString(),
+            repoRev: rev ?? null,
+        })
+            .onConflict((oc) => {
+            const excluded = (col) => ref(`excluded.${col}`);
+            return oc.column('did').doUpdateSet({
+                commitCid: sql `${excluded('commitCid')}`,
+                repoRev: sql `${excluded('repoRev')}`,
+            });
+        })
+            .execute();
+    }
+    async getCurrentRecords(did) {
+        const res = await this.db.db
+            .selectFrom('record')
+            .where('did', '=', did)
+            .select(['uri', 'cid'])
+            .execute();
+        return res.reduce((acc, cur) => {
+            acc[cur.uri] = {
+                uri: new AtUri(cur.uri),
+                cid: CID.parse(cur.cid),
+            };
+            return acc;
+        }, {});
+    }
+    findIndexerForCollection(collection) {
+        const indexers = Object.values(this.records);
+        return indexers.find((indexer) => indexer.collection === collection);
+    }
+    async deleteActor(did) {
+        this.db.assertNotTransaction();
+        const actorIsHosted = await this.getActorIsHosted(did);
+        if (actorIsHosted === false) {
+            await this.db.db.deleteFrom('actor').where('did', '=', did).execute();
+            await this.unindexActor(did);
+        }
+    }
+    async getActorIsHosted(did) {
+        const doc = await this.idResolver.did.resolve(did, true);
+        const pds = doc && getPds(doc);
+        if (!pds)
+            return false;
+        const agent = new Agent(pds);
+        try {
+            await retryXrpc(() => agent.com.atproto.sync.getLatestCommit({ did }));
+            return true;
+        }
+        catch (err) {
+            if (err instanceof ComAtprotoSyncGetLatestCommit.RepoNotFoundError) {
+                return false;
+            }
+            return null;
+        }
+    }
+    async unindexActor(did) {
+        this.db.assertNotTransaction();
+        await this.db.db.deleteFrom('profile').where('creator', '=', did).execute();
+        await this.db.db.deleteFrom('record').where('did', '=', did).execute();
+        const deletedPlaylistIdeas = await this.db.db
+            .deleteFrom('playlist_idea')
+            .where('owner_did', '=', did)
+            .returningAll()
+            .execute();
+        const deletedPlaylistIdeaUris = deletedPlaylistIdeas.map((deletedPlaylistIdea) => {
+            return deletedPlaylistIdea.uri;
+        });
+        const deletedPlaylists = await this.db.db
+            .deleteFrom('playlist')
+            .where('playlist_idea_uri', 'in', deletedPlaylistIdeaUris)
+            .returningAll()
+            .execute();
+        const deletedPlaylistIds = deletedPlaylists.map((deletedPlaylist) => {
+            return deletedPlaylist.id;
+        });
+        await this.db.db
+            .deleteFrom('playlist_item')
+            .where('playlist_id', 'in', deletedPlaylistIds)
+            .execute();
+    }
+}
+const findDiffFromCheckout = (curr, checkout) => {
+    const ops = [];
+    for (const uri of Object.keys(checkout)) {
+        const record = checkout[uri];
+        if (!curr[uri]) {
+            ops.push({ op: 'create', ...record });
+        }
+        else {
+            if (curr[uri].cid.equals(record.cid)) {
+                // no-op
+                continue;
+            }
+            ops.push({ op: 'update', ...record });
+        }
+    }
+    for (const uri of Object.keys(curr)) {
+        const record = curr[uri];
+        if (!checkout[uri]) {
+            ops.push({ op: 'delete', ...record });
+        }
+    }
+    return ops;
+};
+const formatCheckout = (did, verifiedRepo) => {
+    const records = {};
+    for (const create of verifiedRepo.creates) {
+        const uri = AtUri.make(did, create.collection, create.rkey);
+        records[uri.toString()] = {
+            uri,
+            cid: create.cid,
+        };
+    }
+    return records;
+};
+const needsHandleReindex = (actor, timestamp) => {
+    if (!actor)
+        return true;
+    const timeDiff = new Date(timestamp).getTime() - new Date(actor.indexed_at).getTime();
+    // revalidate daily
+    if (timeDiff > DAY)
+        return true;
+    // revalidate more aggressively for invalidated handles
+    if (actor.handle === null && timeDiff > HOUR)
+        return true;
+    return false;
+};

package/dist/dataplane/server/indexing/plugins/playlist-idea.d.ts

@@ -0,0 +1,14 @@
+import { Selectable } from 'kysely';
+import { MasterPlaylist, Record as PlaylistIdeaRecord, RecommendedPlaylist } from '../../../../lexicons/types/com/clioplaylists/alpha/feed/playlistIdea';
+import { Database } from '../../db';
+import { DatabaseSchemaType } from '../../db/database-schema';
+import { RecordProcessor } from '../processor';
+type PlaylistIdea = Selectable<DatabaseSchemaType['playlist_idea']>;
+type IndexedPlaylistIdea = {
+    playlistIdea: PlaylistIdea;
+    masterPlaylist?: MasterPlaylist;
+    recommendedPlaylist?: RecommendedPlaylist;
+};
+export type PluginType = RecordProcessor<PlaylistIdeaRecord, IndexedPlaylistIdea>;
+export declare const makePlugin: (db: Database) => PluginType;
+export default makePlugin;

package/dist/dataplane/server/indexing/plugins/playlist-idea.js

@@ -0,0 +1,126 @@
+import { normalizeDatetimeAlways } from '@atproto/syntax';
+import * as lex from '../../../../lexicons/lexicons';
+import { RecordProcessor } from '../processor';
+const lexId = lex.ids.ComClioplaylistsAlphaFeedPlaylistIdea;
+const insertFn = async (db, uri, cid, obj, timestamp) => {
+    const masterPlaylist = obj.masterPlaylist;
+    const recommendedPlaylist = obj.recommendedPlaylist;
+    let masterPlaylistId;
+    let recommendedPlaylistId;
+    if (masterPlaylist) {
+        const playlist = {
+            playlist_idea_uri: uri.toString(),
+            type: 'master',
+        };
+        const insertedPlaylistIdResponse = await db
+            .insertInto('playlist')
+            .values(playlist)
+            .onConflict((oc) => oc.doNothing())
+            .returning('id')
+            .executeTakeFirst();
+        let songs = masterPlaylist.songs.map((song) => {
+            return {
+                track_name: song.trackName,
+                track_mb_id: song.trackMbId,
+                album_artwork_ref: song.albumArtwork?.ref.toString(),
+                created_at: song.createdAt,
+            };
+        });
+        const insertedSongIdsResponse = await db
+            .insertInto('song')
+            .values(songs)
+            .onConflict((oc) => oc.doNothing())
+            .returning('id')
+            .execute();
+        if (insertedPlaylistIdResponse?.id) {
+            const playlistItems = insertedSongIdsResponse.map((songIdResponse, index) => {
+                return {
+                    playlist_id: insertedPlaylistIdResponse.id,
+                    song_id: songIdResponse.id,
+                    position: index + 1,
+                };
+            });
+            await db.insertInto('playlist_item').values(playlistItems).execute();
+        }
+    }
+    if (recommendedPlaylist) {
+        const playlist = {
+            playlist_idea_uri: uri.toString(),
+            type: 'recommended',
+        };
+        await db
+            .insertInto('playlist')
+            .values(playlist)
+            .onConflict((oc) => oc.doNothing())
+            .execute();
+    }
+    const playlistIdea = {
+        uri: uri.toString(),
+        title: obj.title,
+        description: obj.description,
+        creator_did: obj.creatorDid,
+        owner_did: obj.ownerDid,
+        master_playlist_id: masterPlaylistId,
+        recommended_playlist_id: recommendedPlaylistId,
+        created_at: normalizeDatetimeAlways(obj.createdAt),
+    };
+    const [insertedPlaylistIdea] = await Promise.all([
+        db
+            .insertInto('playlist_idea')
+            .values(playlistIdea)
+            .onConflict((oc) => oc.doNothing())
+            .returningAll()
+            .executeTakeFirst(),
+    ]);
+    if (!insertedPlaylistIdea) {
+        return null; // PlaylistIdea already indexed
+    }
+    return {
+        playlistIdea: insertedPlaylistIdea,
+        masterPlaylist: masterPlaylist,
+        recommendedPlaylist: recommendedPlaylist,
+    };
+};
+const findDuplicate = async () => {
+    return null;
+};
+const deleteFn = async (db, uri) => {
+    const uriStr = uri.toString();
+    const [deletedPlaylistIdea, deletedPlaylists] = await Promise.all([
+        db
+            .deleteFrom('playlist_idea')
+            .where('uri', '=', uriStr)
+            .returningAll()
+            .executeTakeFirst(),
+        db
+            .deleteFrom('playlist')
+            .where('playlist_idea_uri', '=', uriStr)
+            .returningAll()
+            .execute(),
+    ]);
+    const deletedPlaylistIds = deletedPlaylists.map((deletedPlaylist) => {
+        return deletedPlaylist.id;
+    });
+    if (deletedPlaylists) {
+        await db
+            .deleteFrom('playlist_item')
+            .where('playlist_id', 'in', deletedPlaylistIds)
+            .execute();
+    }
+    return deletedPlaylistIdea
+        ? {
+            playlistIdea: deletedPlaylistIdea,
+            masterPlaylist: undefined, // TODO: Figure out if this should be undefined
+            recommendedPlaylist: undefined, // TODO: Figure out if this should be undefined
+        }
+        : null;
+};
+export const makePlugin = (db) => {
+    return new RecordProcessor(db, {
+        lexId,
+        insertFn,
+        findDuplicate,
+        deleteFn,
+    });
+};
+export default makePlugin;

package/dist/dataplane/server/indexing/plugins/profile.d.ts

@@ -0,0 +1,8 @@
+import * as Profile from '../../../../lexicons/types/com/clioplaylists/alpha/actor/profile';
+import { Database } from '../../db';
+import { DatabaseSchemaType } from '../../db/database-schema';
+import { RecordProcessor } from '../processor';
+type IndexedProfile = DatabaseSchemaType['profile'];
+export type PluginType = RecordProcessor<Profile.Record, IndexedProfile>;
+export declare const makePlugin: (db: Database) => PluginType;
+export default makePlugin;

package/dist/dataplane/server/indexing/plugins/profile.js

@@ -0,0 +1,44 @@
+import * as lex from '../../../../lexicons/lexicons';
+import { RecordProcessor } from '../processor';
+const lexId = lex.ids.ComClioplaylistsAlphaActorProfile;
+const insertFn = async (db, uri, cid, obj, timestamp) => {
+    if (uri.rkey !== 'self')
+        return null;
+    const inserted = await db
+        .insertInto('profile')
+        .values({
+        uri: uri.toString(),
+        cid: cid.toString(),
+        creator: uri.host,
+        display_name: obj.displayName,
+        description: obj.description,
+        avatar_cid: obj.avatar?.ref.toString(),
+        banner_cid: obj.banner?.ref.toString(),
+        created_at: obj.createdAt ?? new Date().toISOString(),
+        indexed_at: timestamp,
+    })
+        .onConflict((oc) => oc.doNothing())
+        .returningAll()
+        .executeTakeFirst();
+    return inserted || null;
+};
+const findDuplicate = async () => {
+    return null;
+};
+const deleteFn = async (db, uri) => {
+    const deleted = await db
+        .deleteFrom('profile')
+        .where('uri', '=', uri.toString())
+        .returningAll()
+        .executeTakeFirst();
+    return deleted || null;
+};
+export const makePlugin = (db) => {
+    return new RecordProcessor(db, {
+        lexId,
+        insertFn,
+        findDuplicate,
+        deleteFn,
+    });
+};
+export default makePlugin;

package/dist/dataplane/server/indexing/processor.d.ts

@@ -0,0 +1,22 @@
+import { AtUri } from '@atproto/syntax';
+import { CID } from 'multiformats/cid';
+import { Database } from '../db';
+import { DatabaseSchema } from '../db/database-schema';
+type RecordProcessorParams<T, S> = {
+    lexId: string;
+    insertFn: (db: DatabaseSchema, uri: AtUri, cid: CID, obj: T, timestamp: string) => Promise<S | null>;
+    findDuplicate: (db: DatabaseSchema, uri: AtUri, obj: T) => Promise<AtUri | null>;
+    deleteFn: (db: DatabaseSchema, uri: AtUri) => Promise<S | null>;
+};
+export declare class RecordProcessor<T, S> {
+    private params;
+    collection: string;
+    db: DatabaseSchema;
+    constructor(appDb: Database, params: RecordProcessorParams<T, S>);
+    matchesSchema(obj: unknown): obj is T;
+    assertValidRecord(obj: unknown): asserts obj is T;
+    insertRecord(uri: AtUri, cid: CID, obj: unknown, timestamp: string): Promise<void>;
+    updateRecord(uri: AtUri, cid: CID, obj: unknown, timestamp: string): Promise<void>;
+    deleteRecord(uri: AtUri, cascading?: boolean): Promise<void>;
+}
+export default RecordProcessor;

package/dist/dataplane/server/indexing/processor.js

@@ -0,0 +1,72 @@
+import { stringifyLex } from '@atproto/lexicon';
+import { lexicons } from '../../../lexicons/lexicons';
+export class RecordProcessor {
+    params;
+    collection;
+    db;
+    constructor(appDb, params) {
+        this.params = params;
+        this.db = appDb.db;
+        this.collection = this.params.lexId;
+    }
+    matchesSchema(obj) {
+        try {
+            this.assertValidRecord(obj);
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+    assertValidRecord(obj) {
+        lexicons.assertValidRecord(this.params.lexId, obj);
+    }
+    async insertRecord(uri, cid, obj, timestamp) {
+        this.assertValidRecord(obj);
+        await this.db
+            .insertInto('record')
+            .values({
+            uri: uri.toString(),
+            cid: cid.toString(),
+            did: uri.host,
+            json: stringifyLex(obj),
+            indexed_at: timestamp,
+        })
+            .onConflict((oc) => oc.doNothing())
+            .execute();
+        await this.params.insertFn(this.db, uri, cid, obj, timestamp);
+    }
+    // Currently using a very simple strategy for updates: purge the existing index
+    // for the uri then replace it. The main upside is that this allows the indexer
+    // for each collection to avoid bespoke logic for in-place updates, which isn't
+    // straightforward in the general case. We still get nice control over notifications.
+    async updateRecord(uri, cid, obj, timestamp) {
+        this.assertValidRecord(obj);
+        await this.db
+            .updateTable('record')
+            .where('uri', '=', uri.toString())
+            .set({
+            cid: cid.toString(),
+            json: stringifyLex(obj),
+            indexed_at: timestamp,
+        })
+            .execute();
+        const deleted = await this.params.deleteFn(this.db, uri);
+        if (!deleted) {
+            // If a record was updated but hadn't been indexed yet, treat it like a plain insert.
+            return this.insertRecord(uri, cid, obj, timestamp);
+        }
+        const inserted = await this.params.insertFn(this.db, uri, cid, obj, timestamp);
+        if (!inserted) {
+            throw new Error('Record update failed: removed from index but could not be replaced');
+        }
+    }
+    async deleteRecord(uri, cascading = false) {
+        await this.db
+            .deleteFrom('record')
+            .where('uri', '=', uri.toString())
+            .execute();
+        await this.params.deleteFn(this.db, uri);
+    }
+}
+export default RecordProcessor;

package/dist/dataplane/server/routes/identity.d.ts

@@ -0,0 +1,6 @@
+import { IdResolver } from '@atproto/identity';
+import { ServiceImpl } from '@connectrpc/connect';
+import { ClioService } from '../../../rpc/clio_connect';
+import { Database } from '../db';
+declare const _default: (_db: Database, idResolver: IdResolver) => Partial<ServiceImpl<typeof ClioService>>;
+export default _default;

package/dist/dataplane/server/routes/identity.js

@@ -0,0 +1,54 @@
+import { getDid, getHandle } from '@atproto/identity';
+import { Timestamp } from '@bufbuild/protobuf';
+import { Code, ConnectError } from '@connectrpc/connect';
+export default (_db, idResolver) => ({
+    async getIdentityByDid(req) {
+        const doc = await idResolver.did.resolve(req.did);
+        if (!doc) {
+            throw new ConnectError('identity not found', Code.NotFound);
+        }
+        return getResultFromDoc(doc);
+    },
+    async getIdentityByHandle(req) {
+        const did = await idResolver.handle.resolve(req.handle);
+        if (!did) {
+            throw new ConnectError('identity not found', Code.NotFound);
+        }
+        const doc = await idResolver.did.resolve(did);
+        if (!doc || did !== getDid(doc)) {
+            throw new ConnectError('identity not found', Code.NotFound);
+        }
+        return getResultFromDoc(doc);
+    },
+});
+const getResultFromDoc = (doc) => {
+    const keys = {};
+    doc.verificationMethod?.forEach((method) => {
+        const id = method.id.split('#').at(1);
+        if (!id)
+            return;
+        keys[id] = {
+            Type: method.type,
+            PublicKeyMultibase: method.publicKeyMultibase || '',
+        };
+    });
+    const services = {};
+    doc.service?.forEach((service) => {
+        const id = service.id.split('#').at(1);
+        if (!id)
+            return;
+        if (typeof service.serviceEndpoint !== 'string')
+            return;
+        services[id] = {
+            Type: service.type,
+            URL: service.serviceEndpoint,
+        };
+    });
+    return {
+        did: getDid(doc),
+        handle: getHandle(doc),
+        keys: Buffer.from(JSON.stringify(keys)),
+        services: Buffer.from(JSON.stringify(services)),
+        updated: Timestamp.fromDate(new Date()),
+    };
+};

package/dist/dataplane/server/routes/index.d.ts

@@ -0,0 +1,5 @@
+import { IdResolver } from '@atproto/identity';
+import { ConnectRouter } from '@connectrpc/connect';
+import { Database } from '../db';
+declare const _default: (db: Database, idResolver: IdResolver) => (router: ConnectRouter) => ConnectRouter;
+export default _default;

package/dist/dataplane/server/routes/index.js

@@ -0,0 +1,14 @@
+import { ClioService } from '../../../rpc/clio_connect';
+import identity from './identity';
+import profile from './profile';
+import records from './records';
+import sync from './sync';
+export default (db, idResolver) => (router) => router.service(ClioService, {
+    ...identity(db, idResolver),
+    ...profile(db),
+    ...records(db),
+    ...sync(db),
+    async healthCheck() {
+        return {};
+    },
+});

package/dist/dataplane/server/routes/profile.d.ts

@@ -0,0 +1,5 @@
+import { ServiceImpl } from '@connectrpc/connect';
+import { ClioService } from '../../../rpc/clio_connect';
+import { Database } from '../db';
+declare const _default: (db: Database) => Partial<ServiceImpl<typeof ClioService>>;
+export default _default;