@rocksky/cli 0.3.0 → 0.3.2

package/dist/drizzle/meta/_journal.json

@@ -8,6 +8,13 @@
       "when": 1768065262210,
       "tag": "0000_parallel_paper_doll",
       "breakpoints": true
+    },
+    {
+      "idx": 1,
+      "version": "6",
+      "when": 1768571013590,
+      "tag": "0001_awesome_gabe_jones",
+      "breakpoints": true
     }
   ]
 }

package/dist/index.js

@@ -22,7 +22,7 @@ import { IdResolver } from '@atproto/identity';
 import { configure, getConsoleSink, getLogger } from '@logtape/logtape';
 import { AtpAgent } from '@atproto/api';
 import { sql, eq, and, or, gte, lte } from 'drizzle-orm';
-import { sqliteTable, integer, text, unique } from 'drizzle-orm/sqlite-core';
+import { sqliteTable, integer, text, index, unique } from 'drizzle-orm/sqlite-core';
 import dotenv from 'dotenv';
 import { cleanEnv, str } from 'envalid';
 import crypto from 'node:crypto';
@@ -33,6 +33,8 @@ import { Lexicons } from '@atproto/lexicon';
 import { TID } from '@atproto/common';
 import { createId } from '@paralleldrive/cuid2';
 import _ from 'lodash';
+import { CarReader } from '@ipld/car';
+import * as cbor from '@ipld/dag-cbor';
 import { table, getBorderCharacters } from 'table';
 import { Command } from 'commander';
 import axios from 'axios';
@@ -1369,33 +1371,41 @@ const albums = sqliteTable("albums", {
   updatedAt: integer("updated_at", { mode: "timestamp" }).notNull().default(sql`(unixepoch())`)
 });
 
-const tracks$1 = sqliteTable("tracks", {
-  id: text("id").primaryKey().notNull(),
-  title: text("title").notNull(),
-  artist: text("artist").notNull(),
-  albumArtist: text("album_artist").notNull(),
-  albumArt: text("album_art"),
-  album: text("album").notNull(),
-  trackNumber: integer("track_number"),
-  duration: integer("duration").notNull(),
-  mbId: text("mb_id").unique(),
-  youtubeLink: text("youtube_link").unique(),
-  spotifyLink: text("spotify_link").unique(),
-  appleMusicLink: text("apple_music_link").unique(),
-  tidalLink: text("tidal_link").unique(),
-  discNumber: integer("disc_number"),
-  lyrics: text("lyrics"),
-  composer: text("composer"),
-  genre: text("genre"),
-  label: text("label"),
-  copyrightMessage: text("copyright_message"),
-  uri: text("uri").unique(),
-  cid: text("cid").unique().notNull(),
-  albumUri: text("album_uri"),
-  artistUri: text("artist_uri"),
-  createdAt: integer("created_at", { mode: "timestamp" }).notNull().default(sql`(unixepoch())`),
-  updatedAt: integer("updated_at", { mode: "timestamp" }).notNull().default(sql`(unixepoch())`)
-});
+const tracks$1 = sqliteTable(
+  "tracks",
+  {
+    id: text("id").primaryKey().notNull(),
+    title: text("title").notNull(),
+    artist: text("artist").notNull(),
+    albumArtist: text("album_artist").notNull(),
+    albumArt: text("album_art"),
+    album: text("album").notNull(),
+    trackNumber: integer("track_number"),
+    duration: integer("duration").notNull(),
+    mbId: text("mb_id").unique(),
+    youtubeLink: text("youtube_link").unique(),
+    spotifyLink: text("spotify_link").unique(),
+    appleMusicLink: text("apple_music_link").unique(),
+    tidalLink: text("tidal_link").unique(),
+    discNumber: integer("disc_number"),
+    lyrics: text("lyrics"),
+    composer: text("composer"),
+    genre: text("genre"),
+    label: text("label"),
+    copyrightMessage: text("copyright_message"),
+    uri: text("uri").unique(),
+    cid: text("cid").unique().notNull(),
+    albumUri: text("album_uri"),
+    artistUri: text("artist_uri"),
+    createdAt: integer("created_at", { mode: "timestamp" }).notNull().default(sql`(unixepoch())`),
+    updatedAt: integer("updated_at", { mode: "timestamp" }).notNull().default(sql`(unixepoch())`)
+  },
+  (t) => ({
+    idx_title_artist_album_albumartist: index(
+      "idx_title_artist_album_albumartist"
+    ).on(t.title, t.artist, t.album, t.albumArtist)
+  })
+);
 
 const albumTracks = sqliteTable(
   "album_tracks",
@@ -7566,7 +7576,6 @@ function cleanUpSyncLockOnExit(did) {
   });
 }
 
-const PAGE_SIZE = 100;
 async function sync() {
   const [did, handle] = await getDidAndHandle();
   const agent = await createAgent(did, handle);
@@ -7574,11 +7583,12 @@ async function sync() {
   await subscribeToJetstream(user);
   logger.info`  DID: ${did}`;
   logger.info`  Handle: ${handle}`;
+  const carReader = await downloadCarFile(agent);
   const [artists, albums, songs, scrobbles] = await Promise.all([
-    getRockskyUserArtists(agent),
-    getRockskyUserAlbums(agent),
-    getRockskyUserSongs(agent),
-    getRockskyUserScrobbles(agent)
+    getRockskyUserArtists(agent, carReader),
+    getRockskyUserAlbums(agent, carReader),
+    getRockskyUserSongs(agent, carReader),
+    getRockskyUserScrobbles(agent, carReader)
   ]);
   logger.info`  Artists: ${artists.length}`;
   logger.info`  Albums: ${albums.length}`;
@@ -7636,7 +7646,7 @@ const createArtists = async (artists, user) => {
     id: createId(),
     name: tag
   }));
-  const BATCH_SIZE = 500;
+  const BATCH_SIZE = 1e3;
   for (let i = 0; i < uniqueTags.length; i += BATCH_SIZE) {
     const batch = uniqueTags.slice(i, i + BATCH_SIZE);
     await ctx.db.insert(schema.genres).values(batch).onConflictDoNothing({
@@ -7699,7 +7709,7 @@ const createAlbums = async (albums, user) => {
     )
   );
   const validAlbumData = albums.map((album, index) => ({ album, artist: artists[index] })).filter(({ artist }) => artist);
-  const BATCH_SIZE = 500;
+  const BATCH_SIZE = 1e3;
   let totalAlbumsImported = 0;
   for (let i = 0; i < validAlbumData.length; i += BATCH_SIZE) {
     const batch = validAlbumData.slice(i, i + BATCH_SIZE);
@@ -7759,7 +7769,7 @@ const createSongs = async (songs, user) => {
     artist: artists[index],
     album: albums[index]
   })).filter(({ artist, album }) => artist && album);
-  const BATCH_SIZE = 500;
+  const BATCH_SIZE = 1e3;
   let totalTracksImported = 0;
   for (let i = 0; i < validSongData.length; i += BATCH_SIZE) {
     const batch = validSongData.slice(i, i + BATCH_SIZE);
@@ -7862,7 +7872,7 @@ const createScrobbles = async (scrobbles, user) => {
     album: albums[index],
     artist: artists[index]
   })).filter(({ track, album, artist }) => track && album && artist);
-  const BATCH_SIZE = 500;
+  const BATCH_SIZE = 1e3;
   let totalScrobblesImported = 0;
   for (let i = 0; i < validScrobbleData.length; i += BATCH_SIZE) {
     const batch = validScrobbleData.slice(i, i + BATCH_SIZE);
@@ -7951,7 +7961,7 @@ const subscribeToJetstream = (user) => {
 const onNewCollection = async (record, cid, uri, user) => {
   switch (record.$type) {
     case "app.rocksky.song":
-      await onNewSong(record);
+      await onNewSong(record, cid, uri, user);
       break;
     case "app.rocksky.album":
       await onNewAlbum(record, cid, uri, user);
@@ -7969,6 +7979,16 @@ const onNewCollection = async (record, cid, uri, user) => {
 const onNewSong = async (record, cid, uri, user) => {
   const { title, artist, album } = record;
   logger.info`  New song: ${title} by ${artist} from ${album}`;
+  await createSongs(
+    [
+      {
+        cid,
+        uri,
+        value: record
+      }
+    ],
+    user
+  );
 };
 const onNewAlbum = async (record, cid, uri, user) => {
   const { title, artist } = record;
@@ -7999,8 +8019,140 @@ const onNewArtist = async (record, cid, uri, user) => {
   );
 };
 const onNewScrobble = async (record, cid, uri, user) => {
-  const { title, createdAt } = record;
+  const { title, createdAt, artist, album, albumArtist } = record;
   logger.info`  New scrobble: ${title} at ${createdAt}`;
+  let [artistRecord] = await ctx.db.select().from(schema.artists).where(eq(schema.artists.name, record.albumArtist)).execute();
+  if (!artistRecord) {
+    logger.info`  ⚙️  Artist not found, creating: "${albumArtist}"`;
+    const artistUri = `at://${user.did}/app.rocksky.artist/${createId()}`;
+    const artistCid = createId();
+    await createArtists(
+      [
+        {
+          cid: artistCid,
+          uri: artistUri,
+          value: {
+            $type: "app.rocksky.artist",
+            name: record.albumArtist,
+            createdAt: (/* @__PURE__ */ new Date()).toISOString(),
+            tags: record.tags || []
+          }
+        }
+      ],
+      user
+    );
+    [artistRecord] = await ctx.db.select().from(schema.artists).where(eq(schema.artists.name, record.albumArtist)).execute();
+    if (!artistRecord) {
+      logger.error`  ❌ Failed to create artist. Skipping scrobble.`;
+      return;
+    }
+  }
+  let [albumRecord] = await ctx.db.select().from(schema.albums).where(
+    and(
+      eq(schema.albums.title, record.album),
+      eq(schema.albums.artist, record.albumArtist)
+    )
+  ).execute();
+  if (!albumRecord) {
+    logger.info`  ⚙️  Album not found, creating: "${album}" by ${albumArtist}`;
+    const albumUri = `at://${user.did}/app.rocksky.album/${createId()}`;
+    const albumCid = createId();
+    await createAlbums(
+      [
+        {
+          cid: albumCid,
+          uri: albumUri,
+          value: {
+            $type: "app.rocksky.album",
+            title: record.album,
+            artist: record.albumArtist,
+            createdAt: (/* @__PURE__ */ new Date()).toISOString(),
+            releaseDate: record.releaseDate,
+            year: record.year,
+            albumArt: record.albumArt,
+            artistUri: artistRecord.uri,
+            spotifyLink: record.spotifyLink,
+            appleMusicLink: record.appleMusicLink,
+            tidalLink: record.tidalLink,
+            youtubeLink: record.youtubeLink
+          }
+        }
+      ],
+      user
+    );
+    [albumRecord] = await ctx.db.select().from(schema.albums).where(
+      and(
+        eq(schema.albums.title, record.album),
+        eq(schema.albums.artist, record.albumArtist)
+      )
+    ).execute();
+    if (!albumRecord) {
+      logger.error`  ❌ Failed to create album. Skipping scrobble.`;
+      return;
+    }
+  }
+  let [track] = await ctx.db.select().from(schema.tracks).where(
+    and(
+      eq(schema.tracks.title, record.title),
+      eq(schema.tracks.artist, record.artist),
+      eq(schema.tracks.album, record.album),
+      eq(schema.tracks.albumArtist, record.albumArtist)
+    )
+  ).execute();
+  if (!track) {
+    logger.info`  ⚙️  Track not found, creating: "${title}" by ${artist} from ${album}`;
+    const trackUri = `at://${user.did}/app.rocksky.song/${createId()}`;
+    const trackCid = createId();
+    await createSongs(
+      [
+        {
+          cid: trackCid,
+          uri: trackUri,
+          value: {
+            $type: "app.rocksky.song",
+            title: record.title,
+            artist: record.artist,
+            albumArtist: record.albumArtist,
+            album: record.album,
+            duration: record.duration,
+            trackNumber: record.trackNumber,
+            discNumber: record.discNumber,
+            releaseDate: record.releaseDate,
+            year: record.year,
+            genre: record.genre,
+            tags: record.tags,
+            composer: record.composer,
+            lyrics: record.lyrics,
+            copyrightMessage: record.copyrightMessage,
+            albumArt: record.albumArt,
+            youtubeLink: record.youtubeLink,
+            spotifyLink: record.spotifyLink,
+            tidalLink: record.tidalLink,
+            appleMusicLink: record.appleMusicLink,
+            createdAt: (/* @__PURE__ */ new Date()).toISOString(),
+            mbId: record.mbid,
+            label: record.label,
+            albumUri: albumRecord.uri,
+            artistUri: artistRecord.uri
+          }
+        }
+      ],
+      user
+    );
+    [track] = await ctx.db.select().from(schema.tracks).where(
+      and(
+        eq(schema.tracks.title, record.title),
+        eq(schema.tracks.artist, record.artist),
+        eq(schema.tracks.album, record.album),
+        eq(schema.tracks.albumArtist, record.albumArtist)
+      )
+    ).execute();
+    if (!track) {
+      logger.error`  ❌ Failed to create track. Skipping scrobble.`;
+      return;
+    }
+  }
+  logger.info`  ✓ All required entities ready. Creating scrobble...`;
   await createScrobbles(
     [
       {
@@ -8012,80 +8164,141 @@ const onNewScrobble = async (record, cid, uri, user) => {
     user
   );
 };
-const getRockskyUserSongs = async (agent) => {
-  let results = [];
-  let cursor;
-  do {
-    const res = await agent.com.atproto.repo.listRecords({
-      repo: agent.assertDid,
-      collection: "app.rocksky.song",
-      limit: PAGE_SIZE,
-      cursor
-    });
-    const records = res.data.records;
-    results = results.concat(records);
-    cursor = res.data.cursor;
+const downloadCarFile = async (agent) => {
+  logger.info(`Fetching repository CAR file ...`);
+  const repoRes = await agent.com.atproto.sync.getRepo({
+    did: agent.assertDid
+  });
+  return CarReader.fromBytes(new Uint8Array(repoRes.data));
+};
+const getRockskyUserSongs = async (agent, carReader) => {
+  const results = [];
+  try {
+    const collection = "app.rocksky.song";
+    for await (const { cid, bytes } of carReader.blocks()) {
+      try {
+        const decoded = cbor.decode(bytes);
+        if (decoded && typeof decoded === "object" && "$type" in decoded) {
+          if (decoded.$type === collection) {
+            const value = decoded;
+            const uri = `at://${agent.assertDid}/${collection}/${cid.toString()}`;
+            results.push({
+              value,
+              uri,
+              cid: cid.toString()
+            });
+          }
+        }
+      } catch (e) {
+        logger.warn`  Skipping block with CID ${cid.toString()} due to decode error: ${e}`;
+        continue;
+      }
+    }
     logger.info(
       `${chalk.cyanBright(agent.assertDid)} ${chalk.greenBright(results.length)} songs`
     );
-  } while (cursor);
+  } catch (error) {
+    logger.error(`Error fetching songs from CAR: ${error}`);
+    throw error;
+  }
   return results;
 };
-const getRockskyUserAlbums = async (agent) => {
-  let results = [];
-  let cursor;
-  do {
-    const res = await agent.com.atproto.repo.listRecords({
-      repo: agent.assertDid,
-      collection: "app.rocksky.album",
-      limit: PAGE_SIZE,
-      cursor
-    });
-    const records = res.data.records;
-    results = results.concat(records);
-    cursor = res.data.cursor;
+const getRockskyUserAlbums = async (agent, carReader) => {
+  const results = [];
+  try {
+    const collection = "app.rocksky.album";
+    logger.info`Extracting ${collection} records from CAR file ...`;
+    for await (const { cid, bytes } of carReader.blocks()) {
+      try {
+        const decoded = cbor.decode(bytes);
+        if (decoded && typeof decoded === "object" && "$type" in decoded) {
+          if (decoded.$type === collection) {
+            const value = decoded;
+            const uri = `at://${agent.assertDid}/${collection}/${cid.toString()}`;
+            results.push({
+              value,
+              uri,
+              cid: cid.toString()
+            });
+          }
+        }
+      } catch (e) {
+        logger.warn`  Skipping block with CID ${cid.toString()} due to decode error: ${e}`;
+        continue;
+      }
+    }
     logger.info(
       `${chalk.cyanBright(agent.assertDid)} ${chalk.greenBright(results.length)} albums`
     );
-  } while (cursor);
+  } catch (error) {
+    logger.error(`Error fetching albums from CAR: ${error}`);
+    throw error;
+  }
   return results;
 };
-const getRockskyUserArtists = async (agent) => {
-  let results = [];
-  let cursor;
-  do {
-    const res = await agent.com.atproto.repo.listRecords({
-      repo: agent.assertDid,
-      collection: "app.rocksky.artist",
-      limit: PAGE_SIZE,
-      cursor
-    });
-    const records = res.data.records;
-    results = results.concat(records);
-    cursor = res.data.cursor;
+const getRockskyUserArtists = async (agent, carReader) => {
+  const results = [];
+  try {
+    const collection = "app.rocksky.artist";
+    logger.info`Extracting ${collection} records from CAR file ...`;
+    for await (const { cid, bytes } of carReader.blocks()) {
+      try {
+        const decoded = cbor.decode(bytes);
+        if (decoded && typeof decoded === "object" && "$type" in decoded) {
+          if (decoded.$type === collection) {
+            const value = decoded;
+            const uri = `at://${agent.assertDid}/${collection}/${cid.toString()}`;
+            results.push({
+              value,
+              uri,
+              cid: cid.toString()
+            });
+          }
+        }
+      } catch (e) {
+        continue;
+      }
+    }
     logger.info(
       `${chalk.cyanBright(agent.assertDid)} ${chalk.greenBright(results.length)} artists`
     );
-  } while (cursor);
+  } catch (error) {
+    logger.error(`Error fetching artists from CAR: ${error}`);
+    throw error;
+  }
   return results;
 };
-const getRockskyUserScrobbles = async (agent) => {
-  let results = [];
-  let cursor;
-  do {
-    const res = await agent.com.atproto.repo.listRecords({
-      repo: agent.assertDid,
-      collection: "app.rocksky.scrobble",
-      limit: PAGE_SIZE,
-      cursor
-    });
-    const records = res.data.records;
-    results = results.concat(records);
-    cursor = res.data.cursor;
+const getRockskyUserScrobbles = async (agent, carReader) => {
+  const results = [];
+  try {
+    const collection = "app.rocksky.scrobble";
+    logger.info`Extracting ${collection} records from CAR file ...`;
+    for await (const { cid, bytes } of carReader.blocks()) {
+      try {
+        const decoded = cbor.decode(bytes);
+        if (decoded && typeof decoded === "object" && "$type" in decoded) {
+          if (decoded.$type === collection) {
+            const value = decoded;
+            const uri = `at://${agent.assertDid}/${collection}/${cid.toString()}`;
+            results.push({
+              value,
+              uri,
+              cid: cid.toString()
+            });
+          }
+        }
+      } catch (e) {
+        logger.warn`  Skipping block with CID ${cid.toString()} due to decode error: ${e}`;
+        continue;
+      }
+    }
     logger.info(
       `${chalk.cyanBright(agent.assertDid)} ${chalk.greenBright(results.length)} scrobbles`
     );
-  } while (cursor);
+  } catch (error) {
+    logger.error(`Error fetching scrobbles from CAR: ${error}`);
+    throw error;
+  }
   return results;
 };
 
@@ -8571,7 +8784,7 @@ async function whoami() {
   }
 }
 
-var version = "0.3.0";
+var version = "0.3.2";
 
 async function login(handle) {
   const app = express();
@@ -9201,6 +9414,13 @@ Welcome to the lastfm compatibility API
     return c.text("Scrobble received");
   });
   logger.info`lastfm/listenbrainz/webscrobbler scrobble API listening on ${"http://localhost:" + port}`;
+  new Promise(async () => {
+    try {
+      await sync();
+    } catch (err) {
+      logger.warn`Error during initial sync: ${err}`;
+    }
+  });
   serve({ fetch: app.fetch, port });
 }
 

package/drizzle/0001_awesome_gabe_jones.sql

@@ -0,0 +1 @@
+CREATE INDEX `idx_title_artist_album_albumartist` ON `tracks` (`title`,`artist`,`album`,`album_artist`);