@hasna/experts 0.0.5 → 0.0.6

package/dist/cli/index.js

@@ -1208,6 +1208,34 @@ async function enrichExpert(db, e, opts) {
   }
   return { ok: true, notFound: false, tweets: tweetCount, avatar, rateLimited: false, quotaExhausted: false };
 }
+async function backfillAvatars(db, opts = {}) {
+  const log = opts.onLog ?? (() => {});
+  const fetchFn = opts.fetchFn ?? fetch;
+  const delayMs = opts.delayMs ?? 150;
+  const experts = db.list({ source: opts.source });
+  const res = { downloaded: 0, skipped: 0, failed: 0 };
+  for (const e of experts) {
+    if (e.avatarLocal || !e.avatar) {
+      res.skipped++;
+      continue;
+    }
+    try {
+      const path = await downloadAvatar(e.avatar, e, fetchFn);
+      if (path) {
+        db.setAvatarLocal(e.source, e.sourceId, path);
+        res.downloaded++;
+        if (res.downloaded % 100 === 0)
+          log(`  avatars: ${res.downloaded} downloaded`);
+      } else {
+        res.failed++;
+      }
+    } catch {
+      res.failed++;
+    }
+    await sleep2(delayMs);
+  }
+  return res;
+}
 async function enrichX(db, opts = {}) {
   const log = opts.onLog ?? (() => {});
   const client = opts.client ?? new ConnectorsClient;
@@ -1269,7 +1297,7 @@ async function enrichX(db, opts = {}) {
 }
 
 // src/cli/index.ts
-var VERSION = "0.0.5";
+var VERSION = "0.0.6";
 function openDb() {
   const opts = program.opts();
   return new ExpertsDB(opts.db || defaultDbPath());
@@ -1333,6 +1361,21 @@ program.command("enrich [source]").description("Enrich experts via X/Twitter: pr
   console.log(chalk2.dim(`progress: ${after.enriched}/${after.withHandle} enriched`));
   db.close();
 });
+program.command("avatars [source]").description("Download + properly name profile pictures for experts missing one").option("--delay <ms>", "delay between downloads", (v) => parseInt(v, 10), 150).action(async (source, cmdOpts) => {
+  const db = openDb();
+  requireData(db);
+  console.error(chalk2.dim("Backfilling profile pictures from source media\u2026"));
+  const res = await backfillAvatars(db, {
+    source,
+    delayMs: cmdOpts.delay,
+    onLog: (m) => process.stderr.write(chalk2.dim(m + `
+`))
+  });
+  console.log(chalk2.green(`\u2713 ${res.downloaded} avatars downloaded`) + chalk2.dim(` (${res.skipped} already had one or no URL, ${res.failed} failed)`));
+  const total = db.enrichmentStats(source).avatars;
+  console.log(chalk2.dim(`total experts with a named avatar: ${total}`));
+  db.close();
+});
 program.command("tweets <idOrSlug>").description("Show an expert's stored recent tweets").option("-s, --source <name>", "disambiguate by source").option("-n, --limit <n>", "max tweets", (v) => parseInt(v, 10), 10).action((idOrSlug, cmdOpts) => {
   const db = openDb();
   const e = db.get(idOrSlug, cmdOpts.source);

package/dist/enrich.d.ts

@@ -41,6 +41,23 @@ export declare function enrichExpert(db: ExpertsDB, e: Expert, opts: EnrichOptio
     rateLimited: boolean;
     quotaExhausted: boolean;
 }>;
+export interface AvatarBackfillResult {
+    downloaded: number;
+    skipped: number;
+    failed: number;
+}
+/**
+ * Download + properly name profile pictures for every expert that doesn't have
+ * a local one yet, using the source's own avatar URL (e.g. intro.co media). This
+ * needs no third-party API, so it completes "a named profile picture for every
+ * expert" even when X enrichment is unavailable.
+ */
+export declare function backfillAvatars(db: ExpertsDB, opts?: {
+    source?: string;
+    delayMs?: number;
+    fetchFn?: typeof fetch;
+    onLog?: (m: string) => void;
+}): Promise<AvatarBackfillResult>;
 /** Resumable, throttled enrichment over all experts with a Twitter handle. */
 export declare function enrichX(db: ExpertsDB, opts?: EnrichOptions): Promise<EnrichResult>;
 //# sourceMappingURL=enrich.d.ts.map

package/dist/enrich.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"enrich.d.ts","sourceRoot":"","sources":["../src/enrich.ts"],"names":[],"mappings":"AAYA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,MAAM,CAAC;AACtC,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAChD,OAAO,KAAK,EAAE,MAAM,EAAmB,MAAM,SAAS,CAAC;AAIvD,wBAAgB,SAAS,IAAI,MAAM,CAElC;AAED,4EAA4E;AAC5E,wBAAgB,cAAc,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,GAAG,UAAU,CAAC,GAAG,MAAM,CAQxF;AAED,oEAAoE;AACpE,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAKtD;AAED,qEAAqE;AACrE,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAEjD;AAED,mFAAmF;AACnF,wBAAsB,cAAc,CAClC,GAAG,EAAE,MAAM,EACX,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE,QAAQ,GAAG,UAAU,GAAG,MAAM,GAAG,UAAU,CAAC,EAC5D,OAAO,GAAE,OAAO,KAAa,GAC5B,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAiBxB;AAED,MAAM,WAAW,aAAa;IAC5B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,MAAM,CAAC,EAAE,gBAAgB,CAAC;IAC1B,OAAO,CAAC,EAAE,OAAO,KAAK,CAAC;IACvB,KAAK,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;CAC/B;AAED,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,OAAO,CAAC;IACtB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,uEAAuE;AACvE,wBAAsB,YAAY,CAChC,EAAE,EAAE,SAAS,EACb,CAAC,EAAE,MAAM,EACT,IAAI,EAAE,aAAa,GAClB,OAAO,CAAC;IAAE,EAAE,EAAE,OAAO,CAAC;IAAC,QAAQ,EAAE,OAAO,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,OAAO,CAAC;IAAC,WAAW,EAAE,OAAO,CAAC;IAAC,cAAc,EAAE,OAAO,CAAA;CAAE,CAAC,CAyE7H;AAED,8EAA8E;AAC9E,wBAAsB,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,GAAE,aAAkB,GAAG,OAAO,CAAC,YAAY,CAAC,CAqE5F"}
+{"version":3,"file":"enrich.d.ts","sourceRoot":"","sources":["../src/enrich.ts"],"names":[],"mappings":"AAYA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,MAAM,CAAC;AACtC,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAChD,OAAO,KAAK,EAAE,MAAM,EAAmB,MAAM,SAAS,CAAC;AAIvD,wBAAgB,SAAS,IAAI,MAAM,CAElC;AAED,4EAA4E;AAC5E,wBAAgB,cAAc,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,GAAG,UAAU,CAAC,GAAG,MAAM,CAQxF;AAED,oEAAoE;AACpE,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAKtD;AAED,qEAAqE;AACrE,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAEjD;AAED,mFAAmF;AACnF,wBAAsB,cAAc,CAClC,GAAG,EAAE,MAAM,EACX,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE,QAAQ,GAAG,UAAU,GAAG,MAAM,GAAG,UAAU,CAAC,EAC5D,OAAO,GAAE,OAAO,KAAa,GAC5B,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAiBxB;AAED,MAAM,WAAW,aAAa;IAC5B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,MAAM,CAAC,EAAE,gBAAgB,CAAC;IAC1B,OAAO,CAAC,EAAE,OAAO,KAAK,CAAC;IACvB,KAAK,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;CAC/B;AAED,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,OAAO,CAAC;IACtB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,uEAAuE;AACvE,wBAAsB,YAAY,CAChC,EAAE,EAAE,SAAS,EACb,CAAC,EAAE,MAAM,EACT,IAAI,EAAE,aAAa,GAClB,OAAO,CAAC;IAAE,EAAE,EAAE,OAAO,CAAC;IAAC,QAAQ,EAAE,OAAO,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,OAAO,CAAC;IAAC,WAAW,EAAE,OAAO,CAAC;IAAC,cAAc,EAAE,OAAO,CAAA;CAAE,CAAC,CAyE7H;AAED,MAAM,WAAW,oBAAoB;IACnC,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED;;;;;GAKG;AACH,wBAAsB,eAAe,CACnC,EAAE,EAAE,SAAS,EACb,IAAI,GAAE;IAAE,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,OAAO,KAAK,CAAC;IAAC,KAAK,CAAC,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,IAAI,CAAA;CAAO,GACpG,OAAO,CAAC,oBAAoB,CAAC,CA0B/B;AAED,8EAA8E;AAC9E,wBAAsB,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,GAAE,aAAkB,GAAG,OAAO,CAAC,YAAY,CAAC,CAqE5F"}

package/dist/index.d.ts

@@ -10,7 +10,7 @@ export { crawlSource, type CrawlResult } from "./crawl";
 export { getSource, listSources, registerSource, IntroSource } from "./sources";
 export { IntroClient } from "./sources/intro-api";
 export { normalizeIntroExpert, slugFromUrl } from "./sources/intro";
-export { enrichX, enrichExpert, downloadAvatar, avatarBasename, handleFromSocial, type EnrichOptions, type EnrichResult } from "./enrich";
+export { enrichX, enrichExpert, backfillAvatars, downloadAvatar, avatarBasename, handleFromSocial, type EnrichOptions, type EnrichResult, type AvatarBackfillResult } from "./enrich";
 export { ConnectorsClient, defaultRunner, extractJson, type ConnectorRunner, type ConnectorResult } from "./connectors";
 export { inferTags, expertText } from "./graph";
 export * as format from "./format";

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,cAAc,SAAS,CAAC;AACxB,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,KAAK,WAAW,EAAE,MAAM,MAAM,CAAC;AAClE,OAAO,EAAE,WAAW,EAAE,KAAK,WAAW,EAAE,MAAM,SAAS,CAAC;AACxD,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAChF,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAClD,OAAO,EAAE,oBAAoB,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AACpE,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,cAAc,EAAE,gBAAgB,EAAE,KAAK,aAAa,EAAE,KAAK,YAAY,EAAE,MAAM,UAAU,CAAC;AAC1I,OAAO,EAAE,gBAAgB,EAAE,aAAa,EAAE,WAAW,EAAE,KAAK,eAAe,EAAE,KAAK,eAAe,EAAE,MAAM,cAAc,CAAC;AACxH,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAChD,OAAO,KAAK,MAAM,MAAM,UAAU,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,cAAc,SAAS,CAAC;AACxB,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,KAAK,WAAW,EAAE,MAAM,MAAM,CAAC;AAClE,OAAO,EAAE,WAAW,EAAE,KAAK,WAAW,EAAE,MAAM,SAAS,CAAC;AACxD,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAChF,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAClD,OAAO,EAAE,oBAAoB,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AACpE,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,eAAe,EAAE,cAAc,EAAE,cAAc,EAAE,gBAAgB,EAAE,KAAK,aAAa,EAAE,KAAK,YAAY,EAAE,KAAK,oBAAoB,EAAE,MAAM,UAAU,CAAC;AACtL,OAAO,EAAE,gBAAgB,EAAE,aAAa,EAAE,WAAW,EAAE,KAAK,eAAe,EAAE,KAAK,eAAe,EAAE,MAAM,cAAc,CAAC;AACxH,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAChD,OAAO,KAAK,MAAM,MAAM,UAAU,CAAC"}

package/dist/index.js

@@ -1080,6 +1080,34 @@ async function enrichExpert(db, e, opts) {
   }
   return { ok: true, notFound: false, tweets: tweetCount, avatar, rateLimited: false, quotaExhausted: false };
 }
+async function backfillAvatars(db, opts = {}) {
+  const log = opts.onLog ?? (() => {});
+  const fetchFn = opts.fetchFn ?? fetch;
+  const delayMs = opts.delayMs ?? 150;
+  const experts = db.list({ source: opts.source });
+  const res = { downloaded: 0, skipped: 0, failed: 0 };
+  for (const e of experts) {
+    if (e.avatarLocal || !e.avatar) {
+      res.skipped++;
+      continue;
+    }
+    try {
+      const path = await downloadAvatar(e.avatar, e, fetchFn);
+      if (path) {
+        db.setAvatarLocal(e.source, e.sourceId, path);
+        res.downloaded++;
+        if (res.downloaded % 100 === 0)
+          log(`  avatars: ${res.downloaded} downloaded`);
+      } else {
+        res.failed++;
+      }
+    } catch {
+      res.failed++;
+    }
+    await sleep2(delayMs);
+  }
+  return res;
+}
 async function enrichX(db, opts = {}) {
   const log = opts.onLog ?? (() => {});
   const client = opts.client ?? new ConnectorsClient;
@@ -1781,6 +1809,7 @@ export {
   defaultRunner,
   defaultDbPath,
   crawlSource,
+  backfillAvatars,
   avatarBasename,
   IntroSource,
   IntroClient,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hasna/experts",
-  "version": "0.0.5",
+  "version": "0.0.6",
   "description": "Crawl expert marketplaces (intro.co and more) into a local store, then query them via CLI or a remote HTTP API.",
   "type": "module",
   "main": "dist/index.js",