@rocksky/cli 0.3.0 → 0.3.2

package/drizzle/meta/_journal.json

@@ -8,6 +8,13 @@
       "when": 1768065262210,
       "tag": "0000_parallel_paper_doll",
       "breakpoints": true
+    },
+    {
+      "idx": 1,
+      "version": "6",
+      "when": 1768571013590,
+      "tag": "0001_awesome_gabe_jones",
+      "breakpoints": true
     }
   ]
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rocksky/cli",
-  "version": "0.3.0",
+  "version": "0.3.2",
   "description": "Command-line interface for Rocksky – scrobble tracks, view stats, and manage your listening history",
   "main": "dist/index.js",
   "type": "module",
@@ -32,8 +32,11 @@
     "@atproto/jwk-jose": "0.1.5",
     "@atproto/lex-cli": "^0.5.6",
     "@atproto/lexicon": "^0.4.5",
+    "@atproto/repo": "^0.6.2",
     "@atproto/sync": "^0.1.11",
     "@atproto/syntax": "^0.3.1",
+    "@ipld/car": "^3.2.4",
+    "@ipld/dag-cbor": "^9.2.2",
     "@hono/node-server": "^1.13.8",
     "@logtape/logtape": "^1.3.6",
     "@modelcontextprotocol/sdk": "^1.10.2",
@@ -56,6 +59,7 @@
     "kysely": "^0.27.5",
     "lodash": "^4.17.21",
     "md5": "^2.3.0",
+    "multiformats": "^9.9.0",
     "open": "^10.1.0",
     "table": "^6.9.0",
     "unstorage": "^1.14.4",

package/src/cmd/scrobble-api.ts

@@ -11,6 +11,7 @@ import { matchTrack } from "lib/matchTrack";
 import _ from "lodash";
 import { publishScrobble } from "scrobble";
 import { validateLastfmSignature } from "lib/lastfm";
+import { sync } from "./sync";
 
 export async function scrobbleApi({ port }) {
   const [, handle] = await getDidAndHandle();
@@ -453,5 +454,12 @@ export async function scrobbleApi({ port }) {
 
   log.info`lastfm/listenbrainz/webscrobbler scrobble API listening on ${"http://localhost:" + port}`;
 
+  new Promise(async () => {
+    try {
+      await sync();
+    } catch (err) {
+      log.warn`Error during initial sync: ${err}`;
+    }
+  });
   serve({ fetch: app.fetch, port });
 }

package/src/cmd/sync.ts

@@ -21,8 +21,8 @@ import path from "node:path";
 import { getDidAndHandle } from "lib/getDidAndHandle";
 import { cleanUpJetstreamLockOnExit } from "lib/cleanUpJetstreamLock";
 import { cleanUpSyncLockOnExit } from "lib/cleanUpSyncLock";
-
-const PAGE_SIZE = 100;
+import { CarReader } from "@ipld/car";
+import * as cbor from "@ipld/dag-cbor";
 
 type Artists = { value: Artist.Record; uri: string; cid: string }[];
 type Albums = { value: Album.Record; uri: string; cid: string }[];
@@ -39,11 +39,13 @@ export async function sync() {
   logger.info`  DID: ${did}`;
   logger.info`  Handle: ${handle}`;
 
+  const carReader = await downloadCarFile(agent);
+
   const [artists, albums, songs, scrobbles] = await Promise.all([
-    getRockskyUserArtists(agent),
-    getRockskyUserAlbums(agent),
-    getRockskyUserSongs(agent),
-    getRockskyUserScrobbles(agent),
+    getRockskyUserArtists(agent, carReader),
+    getRockskyUserAlbums(agent, carReader),
+    getRockskyUserSongs(agent, carReader),
+    getRockskyUserScrobbles(agent, carReader),
   ]);
 
   logger.info`  Artists: ${artists.length}`;
@@ -133,7 +135,7 @@ const createArtists = async (artists: Artists, user: SelectUser) => {
       name: tag,
     }));
 
-  const BATCH_SIZE = 500;
+  const BATCH_SIZE = 1000;
   for (let i = 0; i < uniqueTags.length; i += BATCH_SIZE) {
     const batch = uniqueTags.slice(i, i + BATCH_SIZE);
     await ctx.db
@@ -241,7 +243,7 @@ const createAlbums = async (albums: Albums, user: SelectUser) => {
     .filter(({ artist }) => artist);
 
   // Process albums in batches
-  const BATCH_SIZE = 500;
+  const BATCH_SIZE = 1000;
   let totalAlbumsImported = 0;
 
   for (let i = 0; i < validAlbumData.length; i += BATCH_SIZE) {
@@ -332,7 +334,7 @@ const createSongs = async (songs: Songs, user: SelectUser) => {
     .filter(({ artist, album }) => artist && album);
 
   // Process in batches to avoid stack overflow with large datasets
-  const BATCH_SIZE = 500;
+  const BATCH_SIZE = 1000;
   let totalTracksImported = 0;
 
   for (let i = 0; i < validSongData.length; i += BATCH_SIZE) {
@@ -481,7 +483,7 @@ const createScrobbles = async (scrobbles: Scrobbles, user: SelectUser) => {
     .filter(({ track, album, artist }) => track && album && artist);
 
   // Process in batches to avoid stack overflow with large datasets
-  const BATCH_SIZE = 500;
+  const BATCH_SIZE = 1000;
   let totalScrobblesImported = 0;
 
   for (let i = 0; i < validScrobbleData.length; i += BATCH_SIZE) {
@@ -624,6 +626,16 @@ const onNewSong = async (
 ) => {
   const { title, artist, album } = record;
   logger.info`  New song: ${title} by ${artist} from ${album}`;
+  await createSongs(
+    [
+      {
+        cid,
+        uri,
+        value: record,
+      },
+    ],
+    user,
+  );
 };
 
 const onNewAlbum = async (
@@ -672,8 +684,192 @@ const onNewScrobble = async (
   uri: string,
   user: SelectUser,
 ) => {
-  const { title, createdAt } = record;
+  const { title, createdAt, artist, album, albumArtist } = record;
   logger.info`  New scrobble: ${title} at ${createdAt}`;
+
+  // Check if the artist exists, create if not
+  let [artistRecord] = await ctx.db
+    .select()
+    .from(schema.artists)
+    .where(eq(schema.artists.name, record.albumArtist))
+    .execute();
+
+  if (!artistRecord) {
+    logger.info`  ⚙️  Artist not found, creating: "${albumArtist}"`;
+
+    // Create a synthetic artist record from scrobble data
+    const artistUri = `at://${user.did}/app.rocksky.artist/${createId()}`;
+    const artistCid = createId();
+
+    await createArtists(
+      [
+        {
+          cid: artistCid,
+          uri: artistUri,
+          value: {
+            $type: "app.rocksky.artist",
+            name: record.albumArtist,
+            createdAt: new Date().toISOString(),
+            tags: record.tags || [],
+          } as Artist.Record,
+        },
+      ],
+      user,
+    );
+
+    [artistRecord] = await ctx.db
+      .select()
+      .from(schema.artists)
+      .where(eq(schema.artists.name, record.albumArtist))
+      .execute();
+
+    if (!artistRecord) {
+      logger.error`  ❌ Failed to create artist. Skipping scrobble.`;
+      return;
+    }
+  }
+
+  // Check if the album exists, create if not
+  let [albumRecord] = await ctx.db
+    .select()
+    .from(schema.albums)
+    .where(
+      and(
+        eq(schema.albums.title, record.album),
+        eq(schema.albums.artist, record.albumArtist),
+      ),
+    )
+    .execute();
+
+  if (!albumRecord) {
+    logger.info`  ⚙️  Album not found, creating: "${album}" by ${albumArtist}`;
+
+    // Create a synthetic album record from scrobble data
+    const albumUri = `at://${user.did}/app.rocksky.album/${createId()}`;
+    const albumCid = createId();
+
+    await createAlbums(
+      [
+        {
+          cid: albumCid,
+          uri: albumUri,
+          value: {
+            $type: "app.rocksky.album",
+            title: record.album,
+            artist: record.albumArtist,
+            createdAt: new Date().toISOString(),
+            releaseDate: record.releaseDate,
+            year: record.year,
+            albumArt: record.albumArt,
+            artistUri: artistRecord.uri,
+            spotifyLink: record.spotifyLink,
+            appleMusicLink: record.appleMusicLink,
+            tidalLink: record.tidalLink,
+            youtubeLink: record.youtubeLink,
+          } as Album.Record,
+        },
+      ],
+      user,
+    );
+
+    // Fetch the newly created album
+    [albumRecord] = await ctx.db
+      .select()
+      .from(schema.albums)
+      .where(
+        and(
+          eq(schema.albums.title, record.album),
+          eq(schema.albums.artist, record.albumArtist),
+        ),
+      )
+      .execute();
+
+    if (!albumRecord) {
+      logger.error`  ❌ Failed to create album. Skipping scrobble.`;
+      return;
+    }
+  }
+
+  // Check if the track exists, create if not
+  let [track] = await ctx.db
+    .select()
+    .from(schema.tracks)
+    .where(
+      and(
+        eq(schema.tracks.title, record.title),
+        eq(schema.tracks.artist, record.artist),
+        eq(schema.tracks.album, record.album),
+        eq(schema.tracks.albumArtist, record.albumArtist),
+      ),
+    )
+    .execute();
+
+  if (!track) {
+    logger.info`  ⚙️  Track not found, creating: "${title}" by ${artist} from ${album}`;
+
+    // Create a synthetic track record from scrobble data
+    const trackUri = `at://${user.did}/app.rocksky.song/${createId()}`;
+    const trackCid = createId();
+
+    await createSongs(
+      [
+        {
+          cid: trackCid,
+          uri: trackUri,
+          value: {
+            $type: "app.rocksky.song",
+            title: record.title,
+            artist: record.artist,
+            albumArtist: record.albumArtist,
+            album: record.album,
+            duration: record.duration,
+            trackNumber: record.trackNumber,
+            discNumber: record.discNumber,
+            releaseDate: record.releaseDate,
+            year: record.year,
+            genre: record.genre,
+            tags: record.tags,
+            composer: record.composer,
+            lyrics: record.lyrics,
+            copyrightMessage: record.copyrightMessage,
+            albumArt: record.albumArt,
+            youtubeLink: record.youtubeLink,
+            spotifyLink: record.spotifyLink,
+            tidalLink: record.tidalLink,
+            appleMusicLink: record.appleMusicLink,
+            createdAt: new Date().toISOString(),
+            mbId: record.mbid,
+            label: record.label,
+            albumUri: albumRecord.uri,
+            artistUri: artistRecord.uri,
+          } as Song.Record,
+        },
+      ],
+      user,
+    );
+
+    // Fetch the newly created track
+    [track] = await ctx.db
+      .select()
+      .from(schema.tracks)
+      .where(
+        and(
+          eq(schema.tracks.title, record.title),
+          eq(schema.tracks.artist, record.artist),
+          eq(schema.tracks.album, record.album),
+          eq(schema.tracks.albumArtist, record.albumArtist),
+        ),
+      )
+      .execute();
+
+    if (!track) {
+      logger.error`  ❌ Failed to create track. Skipping scrobble.`;
+      return;
+    }
+  }
+
+  logger.info`  ✓ All required entities ready. Creating scrobble...`;
+
   await createScrobbles(
     [
       {
@@ -686,127 +882,201 @@ const onNewScrobble = async (
   );
 };
 
-const getRockskyUserSongs = async (agent: Agent): Promise<Songs> => {
-  let results: {
+const downloadCarFile = async (agent: Agent) => {
+  logger.info(`Fetching repository CAR file ...`);
+
+  const repoRes = await agent.com.atproto.sync.getRepo({
+    did: agent.assertDid,
+  });
+
+  return CarReader.fromBytes(new Uint8Array(repoRes.data));
+};
+
+const getRockskyUserSongs = async (
+  agent: Agent,
+  carReader: CarReader,
+): Promise<Songs> => {
+  const results: {
     value: Song.Record;
     uri: string;
     cid: string;
   }[] = [];
-  let cursor: string | undefined;
-  do {
-    const res = await agent.com.atproto.repo.listRecords({
-      repo: agent.assertDid,
-      collection: "app.rocksky.song",
-      limit: PAGE_SIZE,
-      cursor,
-    });
-    const records = res.data.records as Array<{
-      uri: string;
-      cid: string;
-      value: Song.Record;
-    }>;
-    results = results.concat(records);
-    cursor = res.data.cursor;
+
+  try {
+    const collection = "app.rocksky.song";
+
+    for await (const { cid, bytes } of carReader.blocks()) {
+      try {
+        const decoded = cbor.decode(bytes);
+
+        // Check if this is a record with $type matching our collection
+        if (decoded && typeof decoded === "object" && "$type" in decoded) {
+          if (decoded.$type === collection) {
+            const value = decoded as unknown as Song.Record;
+            // Extract rkey from uri if present in the block, otherwise use cid
+            const uri = `at://${agent.assertDid}/${collection}/${cid.toString()}`;
+
+            results.push({
+              value,
+              uri,
+              cid: cid.toString(),
+            });
+          }
+        }
+      } catch (e) {
+        logger.warn`  Skipping block with CID ${cid.toString()} due to decode error: ${e}`;
+        continue;
+      }
+    }
+
     logger.info(
       `${chalk.cyanBright(agent.assertDid)} ${chalk.greenBright(results.length)} songs`,
     );
-  } while (cursor);
+  } catch (error) {
+    logger.error(`Error fetching songs from CAR: ${error}`);
+    throw error;
+  }
 
   return results;
 };
 
-const getRockskyUserAlbums = async (agent: Agent): Promise<Albums> => {
-  let results: {
+const getRockskyUserAlbums = async (
+  agent: Agent,
+  carReader: CarReader,
+): Promise<Albums> => {
+  const results: {
     value: Album.Record;
     uri: string;
     cid: string;
   }[] = [];
-  let cursor: string | undefined;
-  do {
-    const res = await agent.com.atproto.repo.listRecords({
-      repo: agent.assertDid,
-      collection: "app.rocksky.album",
-      limit: PAGE_SIZE,
-      cursor,
-    });
-
-    const records = res.data.records as Array<{
-      uri: string;
-      cid: string;
-      value: Album.Record;
-    }>;
 
-    results = results.concat(records);
+  try {
+    const collection = "app.rocksky.album";
+    logger.info`Extracting ${collection} records from CAR file ...`;
+
+    for await (const { cid, bytes } of carReader.blocks()) {
+      try {
+        const decoded = cbor.decode(bytes);
+
+        if (decoded && typeof decoded === "object" && "$type" in decoded) {
+          if (decoded.$type === collection) {
+            const value = decoded as unknown as Album.Record;
+            const uri = `at://${agent.assertDid}/${collection}/${cid.toString()}`;
+
+            results.push({
+              value,
+              uri,
+              cid: cid.toString(),
+            });
+          }
+        }
+      } catch (e) {
+        logger.warn`  Skipping block with CID ${cid.toString()} due to decode error: ${e}`;
+        continue;
+      }
+    }
 
-    cursor = res.data.cursor;
     logger.info(
       `${chalk.cyanBright(agent.assertDid)} ${chalk.greenBright(results.length)} albums`,
     );
-  } while (cursor);
+  } catch (error) {
+    logger.error(`Error fetching albums from CAR: ${error}`);
+    throw error;
+  }
 
   return results;
 };
 
-const getRockskyUserArtists = async (agent: Agent): Promise<Artists> => {
-  let results: {
+const getRockskyUserArtists = async (
+  agent: Agent,
+  carReader: CarReader,
+): Promise<Artists> => {
+  const results: {
     value: Artist.Record;
     uri: string;
     cid: string;
   }[] = [];
-  let cursor: string | undefined;
-  do {
-    const res = await agent.com.atproto.repo.listRecords({
-      repo: agent.assertDid,
-      collection: "app.rocksky.artist",
-      limit: PAGE_SIZE,
-      cursor,
-    });
 
-    const records = res.data.records as Array<{
-      uri: string;
-      cid: string;
-      value: Artist.Record;
-    }>;
-
-    results = results.concat(records);
+  try {
+    const collection = "app.rocksky.artist";
+    logger.info`Extracting ${collection} records from CAR file ...`;
+
+    for await (const { cid, bytes } of carReader.blocks()) {
+      try {
+        const decoded = cbor.decode(bytes);
+
+        if (decoded && typeof decoded === "object" && "$type" in decoded) {
+          if (decoded.$type === collection) {
+            const value = decoded as unknown as Artist.Record;
+            const uri = `at://${agent.assertDid}/${collection}/${cid.toString()}`;
+
+            results.push({
+              value,
+              uri,
+              cid: cid.toString(),
+            });
+          }
+        }
+      } catch (e) {
+        // Skip blocks that can't be decoded
+        continue;
+      }
+    }
 
-    cursor = res.data.cursor;
     logger.info(
       `${chalk.cyanBright(agent.assertDid)} ${chalk.greenBright(results.length)} artists`,
     );
-  } while (cursor);
+  } catch (error) {
+    logger.error(`Error fetching artists from CAR: ${error}`);
+    throw error;
+  }
 
   return results;
 };
 
-const getRockskyUserScrobbles = async (agent: Agent): Promise<Scrobbles> => {
-  let results: {
+const getRockskyUserScrobbles = async (
+  agent: Agent,
+  carReader: CarReader,
+): Promise<Scrobbles> => {
+  const results: {
     value: Scrobble.Record;
     uri: string;
     cid: string;
   }[] = [];
-  let cursor: string | undefined;
-  do {
-    const res = await agent.com.atproto.repo.listRecords({
-      repo: agent.assertDid,
-      collection: "app.rocksky.scrobble",
-      limit: PAGE_SIZE,
-      cursor,
-    });
 
-    const records = res.data.records as Array<{
-      uri: string;
-      cid: string;
-      value: Scrobble.Record;
-    }>;
-
-    results = results.concat(records);
+  try {
+    const collection = "app.rocksky.scrobble";
+    logger.info`Extracting ${collection} records from CAR file ...`;
+
+    for await (const { cid, bytes } of carReader.blocks()) {
+      try {
+        const decoded = cbor.decode(bytes);
+
+        if (decoded && typeof decoded === "object" && "$type" in decoded) {
+          if (decoded.$type === collection) {
+            const value = decoded as unknown as Scrobble.Record;
+            const uri = `at://${agent.assertDid}/${collection}/${cid.toString()}`;
+
+            results.push({
+              value,
+              uri,
+              cid: cid.toString(),
+            });
+          }
+        }
+      } catch (e) {
+        logger.warn`  Skipping block with CID ${cid.toString()} due to decode error: ${e}`;
+        continue;
+      }
+    }
 
-    cursor = res.data.cursor;
     logger.info(
       `${chalk.cyanBright(agent.assertDid)} ${chalk.greenBright(results.length)} scrobbles`,
     );
-  } while (cursor);
+  } catch (error) {
+    logger.error(`Error fetching scrobbles from CAR: ${error}`);
+    throw error;
+  }
 
   return results;
 };

package/src/schema/tracks.ts

@@ -1,37 +1,45 @@
 import { type InferInsertModel, type InferSelectModel, sql } from "drizzle-orm";
-import { integer, sqliteTable, text } from "drizzle-orm/sqlite-core";
+import { integer, sqliteTable, text, index } from "drizzle-orm/sqlite-core";
 
-const tracks = sqliteTable("tracks", {
-  id: text("id").primaryKey().notNull(),
-  title: text("title").notNull(),
-  artist: text("artist").notNull(),
-  albumArtist: text("album_artist").notNull(),
-  albumArt: text("album_art"),
-  album: text("album").notNull(),
-  trackNumber: integer("track_number"),
-  duration: integer("duration").notNull(),
-  mbId: text("mb_id").unique(),
-  youtubeLink: text("youtube_link").unique(),
-  spotifyLink: text("spotify_link").unique(),
-  appleMusicLink: text("apple_music_link").unique(),
-  tidalLink: text("tidal_link").unique(),
-  discNumber: integer("disc_number"),
-  lyrics: text("lyrics"),
-  composer: text("composer"),
-  genre: text("genre"),
-  label: text("label"),
-  copyrightMessage: text("copyright_message"),
-  uri: text("uri").unique(),
-  cid: text("cid").unique().notNull(),
-  albumUri: text("album_uri"),
-  artistUri: text("artist_uri"),
-  createdAt: integer("created_at", { mode: "timestamp" })
-    .notNull()
-    .default(sql`(unixepoch())`),
-  updatedAt: integer("updated_at", { mode: "timestamp" })
-    .notNull()
-    .default(sql`(unixepoch())`),
-});
+const tracks = sqliteTable(
+  "tracks",
+  {
+    id: text("id").primaryKey().notNull(),
+    title: text("title").notNull(),
+    artist: text("artist").notNull(),
+    albumArtist: text("album_artist").notNull(),
+    albumArt: text("album_art"),
+    album: text("album").notNull(),
+    trackNumber: integer("track_number"),
+    duration: integer("duration").notNull(),
+    mbId: text("mb_id").unique(),
+    youtubeLink: text("youtube_link").unique(),
+    spotifyLink: text("spotify_link").unique(),
+    appleMusicLink: text("apple_music_link").unique(),
+    tidalLink: text("tidal_link").unique(),
+    discNumber: integer("disc_number"),
+    lyrics: text("lyrics"),
+    composer: text("composer"),
+    genre: text("genre"),
+    label: text("label"),
+    copyrightMessage: text("copyright_message"),
+    uri: text("uri").unique(),
+    cid: text("cid").unique().notNull(),
+    albumUri: text("album_uri"),
+    artistUri: text("artist_uri"),
+    createdAt: integer("created_at", { mode: "timestamp" })
+      .notNull()
+      .default(sql`(unixepoch())`),
+    updatedAt: integer("updated_at", { mode: "timestamp" })
+      .notNull()
+      .default(sql`(unixepoch())`),
+  },
+  (t) => ({
+    idx_title_artist_album_albumartist: index(
+      "idx_title_artist_album_albumartist",
+    ).on(t.title, t.artist, t.album, t.albumArtist),
+  }),
+);
 
 export type SelectTrack = InferSelectModel<typeof tracks>;
 export type InsertTrack = InferInsertModel<typeof tracks>;