@clioplaylists/clio 0.1.1 → 0.1.3

package/dist/dataplane/server/indexing/index.js

@@ -1,321 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || (function () {
-    var ownKeys = function(o) {
-        ownKeys = Object.getOwnPropertyNames || function (o) {
-            var ar = [];
-            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
-            return ar;
-        };
-        return ownKeys(o);
-    };
-    return function (mod) {
-        if (mod && mod.__esModule) return mod;
-        var result = {};
-        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
-        __setModuleDefault(result, mod);
-        return result;
-    };
-})();
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.IndexingService = void 0;
-const api_1 = require("@atproto/api");
-const common_1 = require("@atproto/common");
-const identity_1 = require("@atproto/identity");
-const lexicon_1 = require("@atproto/lexicon");
-const repo_1 = require("@atproto/repo");
-const syntax_1 = require("@atproto/syntax");
-const kysely_1 = require("kysely");
-const cid_1 = require("multiformats/cid");
-const logger_1 = require("../../../logger");
-const retry_1 = require("../../../util/retry");
-const PlaylistIdea = __importStar(require("./plugins/playlist-idea"));
-const Profile = __importStar(require("./plugins/profile"));
-class IndexingService {
-    constructor(db, idResolver) {
-        Object.defineProperty(this, "db", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: db
-        });
-        Object.defineProperty(this, "idResolver", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: idResolver
-        });
-        Object.defineProperty(this, "records", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        this.records = {
-            playlistIdea: PlaylistIdea.makePlugin(this.db),
-            profile: Profile.makePlugin(this.db),
-        };
-    }
-    transact(txn) {
-        txn.assertTransaction();
-        return new IndexingService(txn, this.idResolver);
-    }
-    async indexRecord(uri, cid, obj, action, timestamp) {
-        this.db.assertNotTransaction();
-        await this.db.transaction(async (txn) => {
-            const indexingTx = this.transact(txn);
-            const indexer = indexingTx.findIndexerForCollection(uri.collection);
-            if (!indexer)
-                return;
-            if (action === repo_1.WriteOpAction.Create) {
-                await indexer.insertRecord(uri, cid, obj, timestamp);
-            }
-            else {
-                await indexer.updateRecord(uri, cid, obj, timestamp);
-            }
-        });
-    }
-    async deleteRecord(uri, cascading = false) {
-        this.db.assertNotTransaction();
-        await this.db.transaction(async (txn) => {
-            const indexingTx = this.transact(txn);
-            const indexer = indexingTx.findIndexerForCollection(uri.collection);
-            if (!indexer)
-                return;
-            await indexer.deleteRecord(uri, cascading);
-        });
-    }
-    async indexHandle(did, timestamp, force = false) {
-        this.db.assertNotTransaction();
-        const actor = await this.db.db
-            .selectFrom('actor')
-            .where('did', '=', did)
-            .selectAll()
-            .executeTakeFirst();
-        if (!force && !needsHandleReindex(actor, timestamp)) {
-            return;
-        }
-        const atpData = await this.idResolver.did.resolveAtprotoData(did, true);
-        const handleToDid = await this.idResolver.handle.resolve(atpData.handle);
-        const handle = did === handleToDid ? atpData.handle.toLowerCase() : null;
-        const actorWithHandle = handle !== null
-            ? await this.db.db
-                .selectFrom('actor')
-                .where('handle', '=', handle)
-                .selectAll()
-                .executeTakeFirst()
-            : null;
-        // handle contention
-        if (handle && actorWithHandle && did !== actorWithHandle.did) {
-            await this.db.db
-                .updateTable('actor')
-                .where('actor.did', '=', actorWithHandle.did)
-                .set({ handle: null })
-                .execute();
-        }
-        const actorInfo = { handle, indexed_at: timestamp };
-        await this.db.db
-            .insertInto('actor')
-            .values({ did, ...actorInfo })
-            .onConflict((oc) => oc.column('did').doUpdateSet(actorInfo))
-            .returning('did')
-            .executeTakeFirst();
-    }
-    async indexRepo(did, commit) {
-        this.db.assertNotTransaction();
-        const now = new Date().toISOString();
-        const { pds, signingKey } = await this.idResolver.did.resolveAtprotoData(did, true);
-        const agent = new api_1.Agent(pds);
-        const { data: car } = await (0, retry_1.retryXrpc)(() => agent.com.atproto.sync.getRepo({ did }));
-        const { root, blocks } = await (0, repo_1.readCarWithRoot)(car);
-        const verifiedRepo = await (0, repo_1.verifyRepo)(blocks, root, did, signingKey);
-        const currRecords = await this.getCurrentRecords(did);
-        const repoRecords = formatCheckout(did, verifiedRepo);
-        const diff = findDiffFromCheckout(currRecords, repoRecords);
-        await Promise.all(diff.map(async (op) => {
-            const { uri, cid } = op;
-            try {
-                if (op.op === 'delete') {
-                    await this.deleteRecord(uri);
-                }
-                else {
-                    const parsed = await (0, repo_1.getAndParseRecord)(blocks, cid);
-                    await this.indexRecord(uri, cid, parsed.record, op.op === 'create' ? repo_1.WriteOpAction.Create : repo_1.WriteOpAction.Update, now);
-                }
-            }
-            catch (err) {
-                if (err instanceof lexicon_1.ValidationError) {
-                    logger_1.httpLogger.warn({ did, commit, uri: uri.toString(), cid: cid.toString() }, 'skipping indexing of invalid record');
-                }
-                else {
-                    logger_1.httpLogger.error({ err, did, commit, uri: uri.toString(), cid: cid.toString() }, 'skipping indexing due to error processing record');
-                }
-            }
-        }));
-    }
-    async updateActorStatus(did, active, status = '') {
-        let upstreamStatus;
-        if (active) {
-            upstreamStatus = null;
-        }
-        else if (['deactivated', 'suspended', 'takendown'].includes(status)) {
-            upstreamStatus = status;
-        }
-        else {
-            throw new Error(`Unrecognized account status: ${status}`);
-        }
-        await this.db.db
-            .updateTable('actor')
-            .set({ upstreamStatus })
-            .where('did', '=', did)
-            .execute();
-    }
-    async setCommitLastSeen(did, commit, rev) {
-        const { ref } = this.db.db.dynamic;
-        await this.db.db
-            .insertInto('actor_sync')
-            .values({
-            did,
-            commitCid: commit.toString(),
-            repoRev: rev ?? null,
-        })
-            .onConflict((oc) => {
-            const excluded = (col) => ref(`excluded.${col}`);
-            return oc.column('did').doUpdateSet({
-                commitCid: (0, kysely_1.sql) `${excluded('commitCid')}`,
-                repoRev: (0, kysely_1.sql) `${excluded('repoRev')}`,
-            });
-        })
-            .execute();
-    }
-    async getCurrentRecords(did) {
-        const res = await this.db.db
-            .selectFrom('record')
-            .where('did', '=', did)
-            .select(['uri', 'cid'])
-            .execute();
-        return res.reduce((acc, cur) => {
-            acc[cur.uri] = {
-                uri: new syntax_1.AtUri(cur.uri),
-                cid: cid_1.CID.parse(cur.cid),
-            };
-            return acc;
-        }, {});
-    }
-    findIndexerForCollection(collection) {
-        const indexers = Object.values(this.records);
-        return indexers.find((indexer) => indexer.collection === collection);
-    }
-    async deleteActor(did) {
-        this.db.assertNotTransaction();
-        const actorIsHosted = await this.getActorIsHosted(did);
-        if (actorIsHosted === false) {
-            await this.db.db.deleteFrom('actor').where('did', '=', did).execute();
-            await this.unindexActor(did);
-        }
-    }
-    async getActorIsHosted(did) {
-        const doc = await this.idResolver.did.resolve(did, true);
-        const pds = doc && (0, identity_1.getPds)(doc);
-        if (!pds)
-            return false;
-        const agent = new api_1.Agent(pds);
-        try {
-            await (0, retry_1.retryXrpc)(() => agent.com.atproto.sync.getLatestCommit({ did }));
-            return true;
-        }
-        catch (err) {
-            if (err instanceof api_1.ComAtprotoSyncGetLatestCommit.RepoNotFoundError) {
-                return false;
-            }
-            return null;
-        }
-    }
-    async unindexActor(did) {
-        this.db.assertNotTransaction();
-        await this.db.db.deleteFrom('profile').where('creator', '=', did).execute();
-        await this.db.db.deleteFrom('record').where('did', '=', did).execute();
-        const deletedPlaylistIdeas = await this.db.db
-            .deleteFrom('playlist_idea')
-            .where('owner_did', '=', did)
-            .returningAll()
-            .execute();
-        const deletedPlaylistIdeaUris = deletedPlaylistIdeas.map((deletedPlaylistIdea) => {
-            return deletedPlaylistIdea.uri;
-        });
-        const deletedPlaylists = await this.db.db
-            .deleteFrom('playlist')
-            .where('playlist_idea_uri', 'in', deletedPlaylistIdeaUris)
-            .returningAll()
-            .execute();
-        const deletedPlaylistIds = deletedPlaylists.map((deletedPlaylist) => {
-            return deletedPlaylist.id;
-        });
-        await this.db.db
-            .deleteFrom('playlist_item')
-            .where('playlist_id', 'in', deletedPlaylistIds)
-            .execute();
-    }
-}
-exports.IndexingService = IndexingService;
-const findDiffFromCheckout = (curr, checkout) => {
-    const ops = [];
-    for (const uri of Object.keys(checkout)) {
-        const record = checkout[uri];
-        if (!curr[uri]) {
-            ops.push({ op: 'create', ...record });
-        }
-        else {
-            if (curr[uri].cid.equals(record.cid)) {
-                // no-op
-                continue;
-            }
-            ops.push({ op: 'update', ...record });
-        }
-    }
-    for (const uri of Object.keys(curr)) {
-        const record = curr[uri];
-        if (!checkout[uri]) {
-            ops.push({ op: 'delete', ...record });
-        }
-    }
-    return ops;
-};
-const formatCheckout = (did, verifiedRepo) => {
-    const records = {};
-    for (const create of verifiedRepo.creates) {
-        const uri = syntax_1.AtUri.make(did, create.collection, create.rkey);
-        records[uri.toString()] = {
-            uri,
-            cid: create.cid,
-        };
-    }
-    return records;
-};
-const needsHandleReindex = (actor, timestamp) => {
-    if (!actor)
-        return true;
-    const timeDiff = new Date(timestamp).getTime() - new Date(actor.indexed_at).getTime();
-    // revalidate daily
-    if (timeDiff > common_1.DAY)
-        return true;
-    // revalidate more aggressively for invalidated handles
-    if (actor.handle === null && timeDiff > common_1.HOUR)
-        return true;
-    return false;
-};

package/dist/dataplane/server/indexing/plugins/playlist-idea.d.ts

@@ -1,14 +0,0 @@
-import { Selectable } from 'kysely';
-import { MasterPlaylist, Record as PlaylistIdeaRecord, RecommendedPlaylist } from '../../../../lexicons/types/com/clioplaylists/alpha/feed/playlistIdea';
-import { Database } from '../../db';
-import { DatabaseSchemaType } from '../../db/database-schema';
-import { RecordProcessor } from '../processor';
-type PlaylistIdea = Selectable<DatabaseSchemaType['playlist_idea']>;
-type IndexedPlaylistIdea = {
-    playlistIdea: PlaylistIdea;
-    masterPlaylist?: MasterPlaylist;
-    recommendedPlaylist?: RecommendedPlaylist;
-};
-export type PluginType = RecordProcessor<PlaylistIdeaRecord, IndexedPlaylistIdea>;
-export declare const makePlugin: (db: Database) => PluginType;
-export default makePlugin;

package/dist/dataplane/server/indexing/plugins/playlist-idea.js

@@ -1,163 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || (function () {
-    var ownKeys = function(o) {
-        ownKeys = Object.getOwnPropertyNames || function (o) {
-            var ar = [];
-            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
-            return ar;
-        };
-        return ownKeys(o);
-    };
-    return function (mod) {
-        if (mod && mod.__esModule) return mod;
-        var result = {};
-        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
-        __setModuleDefault(result, mod);
-        return result;
-    };
-})();
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.makePlugin = void 0;
-const syntax_1 = require("@atproto/syntax");
-const lex = __importStar(require("../../../../lexicons/lexicons"));
-const processor_1 = require("../processor");
-const lexId = lex.ids.ComClioplaylistsAlphaFeedPlaylistIdea;
-const insertFn = async (db, uri, cid, obj, timestamp) => {
-    const masterPlaylist = obj.masterPlaylist;
-    const recommendedPlaylist = obj.recommendedPlaylist;
-    let masterPlaylistId;
-    let recommendedPlaylistId;
-    if (masterPlaylist) {
-        const playlist = {
-            playlist_idea_uri: uri.toString(),
-            type: 'master',
-        };
-        const insertedPlaylistIdResponse = await db
-            .insertInto('playlist')
-            .values(playlist)
-            .onConflict((oc) => oc.doNothing())
-            .returning('id')
-            .executeTakeFirst();
-        let songs = masterPlaylist.songs.map((song) => {
-            return {
-                track_name: song.trackName,
-                track_mb_id: song.trackMbId,
-                album_artwork_ref: song.albumArtwork?.ref.toString(),
-                created_at: song.createdAt,
-            };
-        });
-        const insertedSongIdsResponse = await db
-            .insertInto('song')
-            .values(songs)
-            .onConflict((oc) => oc.doNothing())
-            .returning('id')
-            .execute();
-        if (insertedPlaylistIdResponse?.id) {
-            const playlistItems = insertedSongIdsResponse.map((songIdResponse, index) => {
-                return {
-                    playlist_id: insertedPlaylistIdResponse.id,
-                    song_id: songIdResponse.id,
-                    position: index + 1,
-                };
-            });
-            await db.insertInto('playlist_item').values(playlistItems).execute();
-        }
-    }
-    if (recommendedPlaylist) {
-        const playlist = {
-            playlist_idea_uri: uri.toString(),
-            type: 'recommended',
-        };
-        await db
-            .insertInto('playlist')
-            .values(playlist)
-            .onConflict((oc) => oc.doNothing())
-            .execute();
-    }
-    const playlistIdea = {
-        uri: uri.toString(),
-        title: obj.title,
-        description: obj.description,
-        creator_did: obj.creatorDid,
-        owner_did: obj.ownerDid,
-        master_playlist_id: masterPlaylistId,
-        recommended_playlist_id: recommendedPlaylistId,
-        created_at: (0, syntax_1.normalizeDatetimeAlways)(obj.createdAt),
-    };
-    const [insertedPlaylistIdea] = await Promise.all([
-        db
-            .insertInto('playlist_idea')
-            .values(playlistIdea)
-            .onConflict((oc) => oc.doNothing())
-            .returningAll()
-            .executeTakeFirst(),
-    ]);
-    if (!insertedPlaylistIdea) {
-        return null; // PlaylistIdea already indexed
-    }
-    return {
-        playlistIdea: insertedPlaylistIdea,
-        masterPlaylist: masterPlaylist,
-        recommendedPlaylist: recommendedPlaylist,
-    };
-};
-const findDuplicate = async () => {
-    return null;
-};
-const deleteFn = async (db, uri) => {
-    const uriStr = uri.toString();
-    const [deletedPlaylistIdea, deletedPlaylists] = await Promise.all([
-        db
-            .deleteFrom('playlist_idea')
-            .where('uri', '=', uriStr)
-            .returningAll()
-            .executeTakeFirst(),
-        db
-            .deleteFrom('playlist')
-            .where('playlist_idea_uri', '=', uriStr)
-            .returningAll()
-            .execute(),
-    ]);
-    const deletedPlaylistIds = deletedPlaylists.map((deletedPlaylist) => {
-        return deletedPlaylist.id;
-    });
-    if (deletedPlaylists) {
-        await db
-            .deleteFrom('playlist_item')
-            .where('playlist_id', 'in', deletedPlaylistIds)
-            .execute();
-    }
-    return deletedPlaylistIdea
-        ? {
-            playlistIdea: deletedPlaylistIdea,
-            masterPlaylist: undefined, // TODO: Figure out if this should be undefined
-            recommendedPlaylist: undefined, // TODO: Figure out if this should be undefined
-        }
-        : null;
-};
-const makePlugin = (db) => {
-    return new processor_1.RecordProcessor(db, {
-        lexId,
-        insertFn,
-        findDuplicate,
-        deleteFn,
-    });
-};
-exports.makePlugin = makePlugin;
-exports.default = exports.makePlugin;

package/dist/dataplane/server/indexing/plugins/profile.d.ts

@@ -1,8 +0,0 @@
-import * as Profile from '../../../../lexicons/types/com/clioplaylists/alpha/actor/profile';
-import { Database } from '../../db';
-import { DatabaseSchemaType } from '../../db/database-schema';
-import { RecordProcessor } from '../processor';
-type IndexedProfile = DatabaseSchemaType['profile'];
-export type PluginType = RecordProcessor<Profile.Record, IndexedProfile>;
-export declare const makePlugin: (db: Database) => PluginType;
-export default makePlugin;

package/dist/dataplane/server/indexing/plugins/profile.js

@@ -1,81 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || (function () {
-    var ownKeys = function(o) {
-        ownKeys = Object.getOwnPropertyNames || function (o) {
-            var ar = [];
-            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
-            return ar;
-        };
-        return ownKeys(o);
-    };
-    return function (mod) {
-        if (mod && mod.__esModule) return mod;
-        var result = {};
-        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
-        __setModuleDefault(result, mod);
-        return result;
-    };
-})();
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.makePlugin = void 0;
-const lex = __importStar(require("../../../../lexicons/lexicons"));
-const processor_1 = require("../processor");
-const lexId = lex.ids.ComClioplaylistsAlphaActorProfile;
-const insertFn = async (db, uri, cid, obj, timestamp) => {
-    if (uri.rkey !== 'self')
-        return null;
-    const inserted = await db
-        .insertInto('profile')
-        .values({
-        uri: uri.toString(),
-        cid: cid.toString(),
-        creator: uri.host,
-        display_name: obj.displayName,
-        description: obj.description,
-        avatar_cid: obj.avatar?.ref.toString(),
-        banner_cid: obj.banner?.ref.toString(),
-        created_at: obj.createdAt ?? new Date().toISOString(),
-        indexed_at: timestamp,
-    })
-        .onConflict((oc) => oc.doNothing())
-        .returningAll()
-        .executeTakeFirst();
-    return inserted || null;
-};
-const findDuplicate = async () => {
-    return null;
-};
-const deleteFn = async (db, uri) => {
-    const deleted = await db
-        .deleteFrom('profile')
-        .where('uri', '=', uri.toString())
-        .returningAll()
-        .executeTakeFirst();
-    return deleted || null;
-};
-const makePlugin = (db) => {
-    return new processor_1.RecordProcessor(db, {
-        lexId,
-        insertFn,
-        findDuplicate,
-        deleteFn,
-    });
-};
-exports.makePlugin = makePlugin;
-exports.default = exports.makePlugin;

package/dist/dataplane/server/indexing/processor.d.ts

@@ -1,22 +0,0 @@
-import { AtUri } from '@atproto/syntax';
-import { CID } from 'multiformats/cid';
-import { Database } from '../db';
-import { DatabaseSchema } from '../db/database-schema';
-type RecordProcessorParams<T, S> = {
-    lexId: string;
-    insertFn: (db: DatabaseSchema, uri: AtUri, cid: CID, obj: T, timestamp: string) => Promise<S | null>;
-    findDuplicate: (db: DatabaseSchema, uri: AtUri, obj: T) => Promise<AtUri | null>;
-    deleteFn: (db: DatabaseSchema, uri: AtUri) => Promise<S | null>;
-};
-export declare class RecordProcessor<T, S> {
-    private params;
-    collection: string;
-    db: DatabaseSchema;
-    constructor(appDb: Database, params: RecordProcessorParams<T, S>);
-    matchesSchema(obj: unknown): obj is T;
-    assertValidRecord(obj: unknown): asserts obj is T;
-    insertRecord(uri: AtUri, cid: CID, obj: unknown, timestamp: string): Promise<void>;
-    updateRecord(uri: AtUri, cid: CID, obj: unknown, timestamp: string): Promise<void>;
-    deleteRecord(uri: AtUri, cascading?: boolean): Promise<void>;
-}
-export default RecordProcessor;