@hasna/experts 0.0.4 → 0.0.6

package/dist/cli/index.js

@@ -983,13 +983,14 @@ function defaultRunner(bin = "connectors") {
     ]);
     const blob = `${out}
 ${err}`;
-    const rateLimited = /429|rate limit|too many requests/i.test(blob);
+    const rateLimited = /429|rate.?limit|too many requests/i.test(blob);
+    const quotaExhausted = /credit|quota|usage ?cap|usagecap|monthly cap|capexceeded/i.test(blob);
     if (code !== 0) {
-      return { success: false, error: (err || out || `exit ${code}`).trim(), rateLimited };
+      return { success: false, error: (err || out || `exit ${code}`).trim(), rateLimited, quotaExhausted };
     }
     const parsed = extractJson(out);
     if (parsed === undefined) {
-      return { success: false, error: `unparseable output: ${out.slice(0, 200)}`, rateLimited };
+      return { success: false, error: `unparseable output: ${out.slice(0, 200)}`, rateLimited, quotaExhausted };
     }
     return { success: true, data: parsed };
   };
@@ -1119,11 +1120,13 @@ async function enrichExpert(db, e, opts) {
   const handle = handleFromSocial(e.socials.twitter || "");
   const now = new Date().toISOString();
   if (!handle)
-    return { ok: false, notFound: true, tweets: 0, avatar: false, rateLimited: false };
+    return { ok: false, notFound: true, tweets: 0, avatar: false, rateLimited: false, quotaExhausted: false };
   const { user, result } = await client.xUser(handle);
   if (!user) {
+    if (result.quotaExhausted)
+      return { ok: false, notFound: false, tweets: 0, avatar: false, rateLimited: false, quotaExhausted: true };
     if (result.rateLimited)
-      return { ok: false, notFound: false, tweets: 0, avatar: false, rateLimited: true };
+      return { ok: false, notFound: false, tweets: 0, avatar: false, rateLimited: true, quotaExhausted: false };
     db.upsertXProfile({
       source: e.source,
       sourceId: e.sourceId,
@@ -1140,7 +1143,7 @@ async function enrichExpert(db, e, opts) {
       worksOn: "",
       enrichedAt: now
     });
-    return { ok: false, notFound: true, tweets: 0, avatar: false, rateLimited: false };
+    return { ok: false, notFound: true, tweets: 0, avatar: false, rateLimited: false, quotaExhausted: false };
   }
   const profile = {
     source: e.source,
@@ -1175,8 +1178,11 @@ async function enrichExpert(db, e, opts) {
       max: opts.tweetMax ?? 100,
       replies: false
     });
+    if (tlResult.quotaExhausted && tweets.length === 0) {
+      return { ok: true, notFound: false, tweets: 0, avatar, rateLimited: false, quotaExhausted: true };
+    }
     if (tlResult.rateLimited && tweets.length === 0) {
-      return { ok: true, notFound: false, tweets: 0, avatar, rateLimited: true };
+      return { ok: true, notFound: false, tweets: 0, avatar, rateLimited: true, quotaExhausted: false };
     }
     const cutoff = Date.now() - (opts.sinceDays ?? 30) * 86400000;
     const recent = tweets.filter((t) => {
@@ -1200,7 +1206,35 @@ async function enrichExpert(db, e, opts) {
     db.replaceTweets(e.source, e.sourceId, rows);
     tweetCount = rows.length;
   }
-  return { ok: true, notFound: false, tweets: tweetCount, avatar, rateLimited: false };
+  return { ok: true, notFound: false, tweets: tweetCount, avatar, rateLimited: false, quotaExhausted: false };
+}
+async function backfillAvatars(db, opts = {}) {
+  const log = opts.onLog ?? (() => {});
+  const fetchFn = opts.fetchFn ?? fetch;
+  const delayMs = opts.delayMs ?? 150;
+  const experts = db.list({ source: opts.source });
+  const res = { downloaded: 0, skipped: 0, failed: 0 };
+  for (const e of experts) {
+    if (e.avatarLocal || !e.avatar) {
+      res.skipped++;
+      continue;
+    }
+    try {
+      const path = await downloadAvatar(e.avatar, e, fetchFn);
+      if (path) {
+        db.setAvatarLocal(e.source, e.sourceId, path);
+        res.downloaded++;
+        if (res.downloaded % 100 === 0)
+          log(`  avatars: ${res.downloaded} downloaded`);
+      } else {
+        res.failed++;
+      }
+    } catch {
+      res.failed++;
+    }
+    await sleep2(delayMs);
+  }
+  return res;
 }
 async function enrichX(db, opts = {}) {
   const log = opts.onLog ?? (() => {});
@@ -1226,6 +1260,13 @@ async function enrichX(db, opts = {}) {
       await sleep2(delayMs);
       continue;
     }
+    if (outcome.quotaExhausted) {
+      res.stoppedEarly = true;
+      res.reason = "X API credits/quota exhausted \u2014 top up or wait for the monthly reset, then re-run the same command to finish the rest";
+      res.attempted--;
+      log(`X API quota exhausted; stopping at ${i}/${targets.length}. ${res.enriched} enriched so far.`);
+      break;
+    }
     if (outcome.rateLimited) {
       consecutiveRateLimits++;
       if (consecutiveRateLimits >= 3) {
@@ -1256,7 +1297,7 @@ async function enrichX(db, opts = {}) {
 }
 
 // src/cli/index.ts
-var VERSION = "0.0.4";
+var VERSION = "0.0.6";
 function openDb() {
   const opts = program.opts();
   return new ExpertsDB(opts.db || defaultDbPath());
@@ -1320,6 +1361,21 @@ program.command("enrich [source]").description("Enrich experts via X/Twitter: pr
   console.log(chalk2.dim(`progress: ${after.enriched}/${after.withHandle} enriched`));
   db.close();
 });
+program.command("avatars [source]").description("Download + properly name profile pictures for experts missing one").option("--delay <ms>", "delay between downloads", (v) => parseInt(v, 10), 150).action(async (source, cmdOpts) => {
+  const db = openDb();
+  requireData(db);
+  console.error(chalk2.dim("Backfilling profile pictures from source media\u2026"));
+  const res = await backfillAvatars(db, {
+    source,
+    delayMs: cmdOpts.delay,
+    onLog: (m) => process.stderr.write(chalk2.dim(m + `
+`))
+  });
+  console.log(chalk2.green(`\u2713 ${res.downloaded} avatars downloaded`) + chalk2.dim(` (${res.skipped} already had one or no URL, ${res.failed} failed)`));
+  const total = db.enrichmentStats(source).avatars;
+  console.log(chalk2.dim(`total experts with a named avatar: ${total}`));
+  db.close();
+});
 program.command("tweets <idOrSlug>").description("Show an expert's stored recent tweets").option("-s, --source <name>", "disambiguate by source").option("-n, --limit <n>", "max tweets", (v) => parseInt(v, 10), 10).action((idOrSlug, cmdOpts) => {
   const db = openDb();
   const e = db.get(idOrSlug, cmdOpts.source);

package/dist/connectors.d.ts

@@ -10,7 +10,10 @@ export interface ConnectorResult {
     success: boolean;
     data?: any;
     error?: string;
+    /** Transient 429 / rate-window limit — back off and retry. */
     rateLimited?: boolean;
+    /** Hard quota/credit exhaustion — retrying won't help until reset/top-up. */
+    quotaExhausted?: boolean;
 }
 /** Runs `connectors run <connector> <args…> --format json`. */
 export type ConnectorRunner = (connector: string, args: string[]) => Promise<ConnectorResult>;

package/dist/connectors.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"connectors.d.ts","sourceRoot":"","sources":["../src/connectors.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,OAAO,CAAC;IACjB,IAAI,CAAC,EAAE,GAAG,CAAC;IACX,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB;AAED,+DAA+D;AAC/D,MAAM,MAAM,eAAe,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,eAAe,CAAC,CAAC;AAE9F,MAAM,WAAW,uBAAuB;IACtC,MAAM,CAAC,EAAE,eAAe,CAAC;IACzB,+CAA+C;IAC/C,GAAG,CAAC,EAAE,MAAM,CAAC;CACd;AAED,mDAAmD;AACnD,wBAAgB,aAAa,CAAC,GAAG,SAAe,GAAG,eAAe,CAuBjE;AAED,gEAAgE;AAChE,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,GAAG,CAqB7C;AAED,MAAM,WAAW,KAAK;IACpB,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,eAAe,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,MAAM;IACrB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe,EAAE,MAAM,CAAC;IACxB,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,GAAG,CAAkB;gBAEjB,IAAI,GAAE,uBAA4B;IAI9C,gEAAgE;IAC1D,KAAK,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,IAAI,EAAE,KAAK,GAAG,IAAI,CAAC;QAAC,MAAM,EAAE,eAAe,CAAA;KAAE,CAAC;IAuBvF,oEAAoE;IAC9D,SAAS,CACb,MAAM,EAAE,MAAM,EACd,IAAI,GAAE;QAAE,GAAG,CAAC,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,OAAO,CAAC;QAAC,QAAQ,CAAC,EAAE,OAAO,CAAA;KAAO,GACjE,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,EAAE,CAAC;QAAC,MAAM,EAAE,eAAe,CAAA;KAAE,CAAC;CAwB1D"}
+{"version":3,"file":"connectors.d.ts","sourceRoot":"","sources":["../src/connectors.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,OAAO,CAAC;IACjB,IAAI,CAAC,EAAE,GAAG,CAAC;IACX,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,8DAA8D;IAC9D,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,6EAA6E;IAC7E,cAAc,CAAC,EAAE,OAAO,CAAC;CAC1B;AAED,+DAA+D;AAC/D,MAAM,MAAM,eAAe,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,eAAe,CAAC,CAAC;AAE9F,MAAM,WAAW,uBAAuB;IACtC,MAAM,CAAC,EAAE,eAAe,CAAC;IACzB,+CAA+C;IAC/C,GAAG,CAAC,EAAE,MAAM,CAAC;CACd;AAED,mDAAmD;AACnD,wBAAgB,aAAa,CAAC,GAAG,SAAe,GAAG,eAAe,CA0BjE;AAED,gEAAgE;AAChE,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,GAAG,CAqB7C;AAED,MAAM,WAAW,KAAK;IACpB,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,eAAe,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,MAAM;IACrB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe,EAAE,MAAM,CAAC;IACxB,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,GAAG,CAAkB;gBAEjB,IAAI,GAAE,uBAA4B;IAI9C,gEAAgE;IAC1D,KAAK,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,IAAI,EAAE,KAAK,GAAG,IAAI,CAAC;QAAC,MAAM,EAAE,eAAe,CAAA;KAAE,CAAC;IAuBvF,oEAAoE;IAC9D,SAAS,CACb,MAAM,EAAE,MAAM,EACd,IAAI,GAAE;QAAE,GAAG,CAAC,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,OAAO,CAAC;QAAC,QAAQ,CAAC,EAAE,OAAO,CAAA;KAAO,GACjE,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,EAAE,CAAC;QAAC,MAAM,EAAE,eAAe,CAAA;KAAE,CAAC;CAwB1D"}

package/dist/enrich.d.ts

@@ -39,7 +39,25 @@ export declare function enrichExpert(db: ExpertsDB, e: Expert, opts: EnrichOptio
     tweets: number;
     avatar: boolean;
     rateLimited: boolean;
+    quotaExhausted: boolean;
 }>;
+export interface AvatarBackfillResult {
+    downloaded: number;
+    skipped: number;
+    failed: number;
+}
+/**
+ * Download + properly name profile pictures for every expert that doesn't have
+ * a local one yet, using the source's own avatar URL (e.g. intro.co media). This
+ * needs no third-party API, so it completes "a named profile picture for every
+ * expert" even when X enrichment is unavailable.
+ */
+export declare function backfillAvatars(db: ExpertsDB, opts?: {
+    source?: string;
+    delayMs?: number;
+    fetchFn?: typeof fetch;
+    onLog?: (m: string) => void;
+}): Promise<AvatarBackfillResult>;
 /** Resumable, throttled enrichment over all experts with a Twitter handle. */
 export declare function enrichX(db: ExpertsDB, opts?: EnrichOptions): Promise<EnrichResult>;
 //# sourceMappingURL=enrich.d.ts.map

package/dist/enrich.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"enrich.d.ts","sourceRoot":"","sources":["../src/enrich.ts"],"names":[],"mappings":"AAYA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,MAAM,CAAC;AACtC,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAChD,OAAO,KAAK,EAAE,MAAM,EAAmB,MAAM,SAAS,CAAC;AAIvD,wBAAgB,SAAS,IAAI,MAAM,CAElC;AAED,4EAA4E;AAC5E,wBAAgB,cAAc,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,GAAG,UAAU,CAAC,GAAG,MAAM,CAQxF;AAED,oEAAoE;AACpE,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAKtD;AAED,qEAAqE;AACrE,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAEjD;AAED,mFAAmF;AACnF,wBAAsB,cAAc,CAClC,GAAG,EAAE,MAAM,EACX,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE,QAAQ,GAAG,UAAU,GAAG,MAAM,GAAG,UAAU,CAAC,EAC5D,OAAO,GAAE,OAAO,KAAa,GAC5B,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAiBxB;AAED,MAAM,WAAW,aAAa;IAC5B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,MAAM,CAAC,EAAE,gBAAgB,CAAC;IAC1B,OAAO,CAAC,EAAE,OAAO,KAAK,CAAC;IACvB,KAAK,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;CAC/B;AAED,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,OAAO,CAAC;IACtB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,uEAAuE;AACvE,wBAAsB,YAAY,CAChC,EAAE,EAAE,SAAS,EACb,CAAC,EAAE,MAAM,EACT,IAAI,EAAE,aAAa,GAClB,OAAO,CAAC;IAAE,EAAE,EAAE,OAAO,CAAC;IAAC,QAAQ,EAAE,OAAO,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,OAAO,CAAC;IAAC,WAAW,EAAE,OAAO,CAAA;CAAE,CAAC,CAoEpG;AAED,8EAA8E;AAC9E,wBAAsB,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,GAAE,aAAkB,GAAG,OAAO,CAAC,YAAY,CAAC,CA4D5F"}
+{"version":3,"file":"enrich.d.ts","sourceRoot":"","sources":["../src/enrich.ts"],"names":[],"mappings":"AAYA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,MAAM,CAAC;AACtC,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAChD,OAAO,KAAK,EAAE,MAAM,EAAmB,MAAM,SAAS,CAAC;AAIvD,wBAAgB,SAAS,IAAI,MAAM,CAElC;AAED,4EAA4E;AAC5E,wBAAgB,cAAc,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,GAAG,UAAU,CAAC,GAAG,MAAM,CAQxF;AAED,oEAAoE;AACpE,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAKtD;AAED,qEAAqE;AACrE,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAEjD;AAED,mFAAmF;AACnF,wBAAsB,cAAc,CAClC,GAAG,EAAE,MAAM,EACX,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE,QAAQ,GAAG,UAAU,GAAG,MAAM,GAAG,UAAU,CAAC,EAC5D,OAAO,GAAE,OAAO,KAAa,GAC5B,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAiBxB;AAED,MAAM,WAAW,aAAa;IAC5B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,MAAM,CAAC,EAAE,gBAAgB,CAAC;IAC1B,OAAO,CAAC,EAAE,OAAO,KAAK,CAAC;IACvB,KAAK,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;CAC/B;AAED,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,OAAO,CAAC;IACtB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,uEAAuE;AACvE,wBAAsB,YAAY,CAChC,EAAE,EAAE,SAAS,EACb,CAAC,EAAE,MAAM,EACT,IAAI,EAAE,aAAa,GAClB,OAAO,CAAC;IAAE,EAAE,EAAE,OAAO,CAAC;IAAC,QAAQ,EAAE,OAAO,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,OAAO,CAAC;IAAC,WAAW,EAAE,OAAO,CAAC;IAAC,cAAc,EAAE,OAAO,CAAA;CAAE,CAAC,CAyE7H;AAED,MAAM,WAAW,oBAAoB;IACnC,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED;;;;;GAKG;AACH,wBAAsB,eAAe,CACnC,EAAE,EAAE,SAAS,EACb,IAAI,GAAE;IAAE,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,OAAO,KAAK,CAAC;IAAC,KAAK,CAAC,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,IAAI,CAAA;CAAO,GACpG,OAAO,CAAC,oBAAoB,CAAC,CA0B/B;AAED,8EAA8E;AAC9E,wBAAsB,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,GAAE,aAAkB,GAAG,OAAO,CAAC,YAAY,CAAC,CAqE5F"}

package/dist/index.d.ts

@@ -10,7 +10,7 @@ export { crawlSource, type CrawlResult } from "./crawl";
 export { getSource, listSources, registerSource, IntroSource } from "./sources";
 export { IntroClient } from "./sources/intro-api";
 export { normalizeIntroExpert, slugFromUrl } from "./sources/intro";
-export { enrichX, enrichExpert, downloadAvatar, avatarBasename, handleFromSocial, type EnrichOptions, type EnrichResult } from "./enrich";
+export { enrichX, enrichExpert, backfillAvatars, downloadAvatar, avatarBasename, handleFromSocial, type EnrichOptions, type EnrichResult, type AvatarBackfillResult } from "./enrich";
 export { ConnectorsClient, defaultRunner, extractJson, type ConnectorRunner, type ConnectorResult } from "./connectors";
 export { inferTags, expertText } from "./graph";
 export * as format from "./format";

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,cAAc,SAAS,CAAC;AACxB,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,KAAK,WAAW,EAAE,MAAM,MAAM,CAAC;AAClE,OAAO,EAAE,WAAW,EAAE,KAAK,WAAW,EAAE,MAAM,SAAS,CAAC;AACxD,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAChF,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAClD,OAAO,EAAE,oBAAoB,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AACpE,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,cAAc,EAAE,gBAAgB,EAAE,KAAK,aAAa,EAAE,KAAK,YAAY,EAAE,MAAM,UAAU,CAAC;AAC1I,OAAO,EAAE,gBAAgB,EAAE,aAAa,EAAE,WAAW,EAAE,KAAK,eAAe,EAAE,KAAK,eAAe,EAAE,MAAM,cAAc,CAAC;AACxH,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAChD,OAAO,KAAK,MAAM,MAAM,UAAU,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,cAAc,SAAS,CAAC;AACxB,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,KAAK,WAAW,EAAE,MAAM,MAAM,CAAC;AAClE,OAAO,EAAE,WAAW,EAAE,KAAK,WAAW,EAAE,MAAM,SAAS,CAAC;AACxD,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAChF,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAClD,OAAO,EAAE,oBAAoB,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AACpE,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,eAAe,EAAE,cAAc,EAAE,cAAc,EAAE,gBAAgB,EAAE,KAAK,aAAa,EAAE,KAAK,YAAY,EAAE,KAAK,oBAAoB,EAAE,MAAM,UAAU,CAAC;AACtL,OAAO,EAAE,gBAAgB,EAAE,aAAa,EAAE,WAAW,EAAE,KAAK,eAAe,EAAE,KAAK,eAAe,EAAE,MAAM,cAAc,CAAC;AACxH,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAChD,OAAO,KAAK,MAAM,MAAM,UAAU,CAAC"}

package/dist/index.js

@@ -855,13 +855,14 @@ function defaultRunner(bin = "connectors") {
     ]);
     const blob = `${out}
 ${err}`;
-    const rateLimited = /429|rate limit|too many requests/i.test(blob);
+    const rateLimited = /429|rate.?limit|too many requests/i.test(blob);
+    const quotaExhausted = /credit|quota|usage ?cap|usagecap|monthly cap|capexceeded/i.test(blob);
     if (code !== 0) {
-      return { success: false, error: (err || out || `exit ${code}`).trim(), rateLimited };
+      return { success: false, error: (err || out || `exit ${code}`).trim(), rateLimited, quotaExhausted };
     }
     const parsed = extractJson(out);
     if (parsed === undefined) {
-      return { success: false, error: `unparseable output: ${out.slice(0, 200)}`, rateLimited };
+      return { success: false, error: `unparseable output: ${out.slice(0, 200)}`, rateLimited, quotaExhausted };
     }
     return { success: true, data: parsed };
   };
@@ -991,11 +992,13 @@ async function enrichExpert(db, e, opts) {
   const handle = handleFromSocial(e.socials.twitter || "");
   const now = new Date().toISOString();
   if (!handle)
-    return { ok: false, notFound: true, tweets: 0, avatar: false, rateLimited: false };
+    return { ok: false, notFound: true, tweets: 0, avatar: false, rateLimited: false, quotaExhausted: false };
   const { user, result } = await client.xUser(handle);
   if (!user) {
+    if (result.quotaExhausted)
+      return { ok: false, notFound: false, tweets: 0, avatar: false, rateLimited: false, quotaExhausted: true };
     if (result.rateLimited)
-      return { ok: false, notFound: false, tweets: 0, avatar: false, rateLimited: true };
+      return { ok: false, notFound: false, tweets: 0, avatar: false, rateLimited: true, quotaExhausted: false };
     db.upsertXProfile({
       source: e.source,
       sourceId: e.sourceId,
@@ -1012,7 +1015,7 @@ async function enrichExpert(db, e, opts) {
       worksOn: "",
       enrichedAt: now
     });
-    return { ok: false, notFound: true, tweets: 0, avatar: false, rateLimited: false };
+    return { ok: false, notFound: true, tweets: 0, avatar: false, rateLimited: false, quotaExhausted: false };
   }
   const profile = {
     source: e.source,
@@ -1047,8 +1050,11 @@ async function enrichExpert(db, e, opts) {
       max: opts.tweetMax ?? 100,
       replies: false
     });
+    if (tlResult.quotaExhausted && tweets.length === 0) {
+      return { ok: true, notFound: false, tweets: 0, avatar, rateLimited: false, quotaExhausted: true };
+    }
     if (tlResult.rateLimited && tweets.length === 0) {
-      return { ok: true, notFound: false, tweets: 0, avatar, rateLimited: true };
+      return { ok: true, notFound: false, tweets: 0, avatar, rateLimited: true, quotaExhausted: false };
     }
     const cutoff = Date.now() - (opts.sinceDays ?? 30) * 86400000;
     const recent = tweets.filter((t) => {
@@ -1072,7 +1078,35 @@ async function enrichExpert(db, e, opts) {
     db.replaceTweets(e.source, e.sourceId, rows);
     tweetCount = rows.length;
   }
-  return { ok: true, notFound: false, tweets: tweetCount, avatar, rateLimited: false };
+  return { ok: true, notFound: false, tweets: tweetCount, avatar, rateLimited: false, quotaExhausted: false };
+}
+async function backfillAvatars(db, opts = {}) {
+  const log = opts.onLog ?? (() => {});
+  const fetchFn = opts.fetchFn ?? fetch;
+  const delayMs = opts.delayMs ?? 150;
+  const experts = db.list({ source: opts.source });
+  const res = { downloaded: 0, skipped: 0, failed: 0 };
+  for (const e of experts) {
+    if (e.avatarLocal || !e.avatar) {
+      res.skipped++;
+      continue;
+    }
+    try {
+      const path = await downloadAvatar(e.avatar, e, fetchFn);
+      if (path) {
+        db.setAvatarLocal(e.source, e.sourceId, path);
+        res.downloaded++;
+        if (res.downloaded % 100 === 0)
+          log(`  avatars: ${res.downloaded} downloaded`);
+      } else {
+        res.failed++;
+      }
+    } catch {
+      res.failed++;
+    }
+    await sleep2(delayMs);
+  }
+  return res;
 }
 async function enrichX(db, opts = {}) {
   const log = opts.onLog ?? (() => {});
@@ -1098,6 +1132,13 @@ async function enrichX(db, opts = {}) {
       await sleep2(delayMs);
       continue;
     }
+    if (outcome.quotaExhausted) {
+      res.stoppedEarly = true;
+      res.reason = "X API credits/quota exhausted \u2014 top up or wait for the monthly reset, then re-run the same command to finish the rest";
+      res.attempted--;
+      log(`X API quota exhausted; stopping at ${i}/${targets.length}. ${res.enriched} enriched so far.`);
+      break;
+    }
     if (outcome.rateLimited) {
       consecutiveRateLimits++;
       if (consecutiveRateLimits >= 3) {
@@ -1768,6 +1809,7 @@ export {
   defaultRunner,
   defaultDbPath,
   crawlSource,
+  backfillAvatars,
   avatarBasename,
   IntroSource,
   IntroClient,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hasna/experts",
-  "version": "0.0.4",
+  "version": "0.0.6",
   "description": "Crawl expert marketplaces (intro.co and more) into a local store, then query them via CLI or a remote HTTP API.",
   "type": "module",
   "main": "dist/index.js",