@nanocollective/roster 0.1.0-beta.2 → 0.1.0-beta.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/cli.js CHANGED
@@ -2913,7 +2913,20 @@ function workerBundle() {
2913
2913
  }
2914
2914
  return null;
2915
2915
  }
2916
- function wrangler(args) {
2916
+ async function wrangler(args) {
2917
+ let code = 1;
2918
+ for (let attempt = 1; attempt <= 3; attempt++) {
2919
+ code = await wranglerOnce(args);
2920
+ if (code === 0) return 0;
2921
+ if (attempt < 3) {
2922
+ process.stdout.write(` wrangler failed; trying again (${attempt + 1} of 3)\u2026
2923
+ `);
2924
+ await new Promise((r) => setTimeout(r, 15e3 * attempt));
2925
+ }
2926
+ }
2927
+ return code;
2928
+ }
2929
+ function wranglerOnce(args) {
2917
2930
  return new Promise((resolve8) => {
2918
2931
  const child = spawn("npx", ["--yes", WRANGLER, ...args], { stdio: "inherit" });
2919
2932
  child.on("close", (code) => resolve8(code ?? 1));
@@ -6922,7 +6935,7 @@ var TIMELINE_PR = `
6922
6935
  ... on ReadyForReviewEvent { actor { login } createdAt }
6923
6936
  ... on ReviewRequestedEvent {
6924
6937
  actor { login } createdAt
6925
- requestedReviewer { ... on User { login } ... on Team { name } }
6938
+ requestedReviewer { __typename ... on User { login } ... on Bot { login } }
6926
6939
  }
6927
6940
  `;
6928
6941
  var CLOSED_FIRST = 30;
@@ -7167,7 +7180,9 @@ function event(e) {
7167
7180
  type: "review-requested",
7168
7181
  actor: who,
7169
7182
  createdAt: at,
7170
- assignee: e.requestedReviewer?.login ?? e.requestedReviewer?.name ?? ""
7183
+ /* A team's name needs the read:org scope, which a portal signed in with `repo` alone does
7184
+ not have, and asking for it fails the whole thread. Its type needs nothing. */
7185
+ assignee: e.requestedReviewer?.login ?? (e.requestedReviewer?.__typename === "Team" ? "a team" : "")
7171
7186
  };
7172
7187
  case "LabeledEvent":
7173
7188
  return { type: "labeled", actor: who, createdAt: at, label: e.label?.name ?? "" };
@@ -7533,15 +7548,19 @@ function isTextPath(path, size2) {
7533
7548
  return size2 <= MAX_TEXT && !BINARY.test(path);
7534
7549
  }
7535
7550
  var Snapshots = class {
7536
- constructor(kv, freshMs = 4e3) {
7551
+ constructor(kv, freshMs = 4e3, shareInflight = true) {
7537
7552
  this.kv = kv;
7538
7553
  this.freshMs = freshMs;
7554
+ this.shareInflight = shareInflight;
7539
7555
  }
7540
7556
  kv;
7541
7557
  freshMs;
7558
+ shareInflight;
7542
7559
  repos = /* @__PURE__ */ new Map();
7543
7560
  inflight = /* @__PURE__ */ new Map();
7544
7561
  checked = /* @__PURE__ */ new Map();
7562
+ /** Who wants to hear how a refresh in flight is going. Every caller that joined it. */
7563
+ listeners = /* @__PURE__ */ new Map();
7545
7564
  get(repo) {
7546
7565
  return this.repos.get(repo);
7547
7566
  }
@@ -7556,11 +7575,23 @@ var Snapshots = class {
7556
7575
  if (!opts.force && this.repos.has(repo) && Date.now() - last < this.freshMs) {
7557
7576
  return Promise.resolve({ ok: true, changed: false });
7558
7577
  }
7578
+ const pathsOnly = opts.pathsOnly === true;
7579
+ if (!this.shareInflight) {
7580
+ return this.load(gh, repo, pathsOnly, (step) => opts.report?.(repo, step));
7581
+ }
7582
+ if (opts.report) {
7583
+ if (!this.listeners.has(repo)) this.listeners.set(repo, /* @__PURE__ */ new Set());
7584
+ this.listeners.get(repo).add(opts.report);
7585
+ }
7559
7586
  const running = this.inflight.get(repo);
7560
7587
  if (running) return running;
7561
- const work = this.load(gh, repo, opts.pathsOnly === true).finally(
7562
- () => this.inflight.delete(repo)
7563
- );
7588
+ const say = (step) => {
7589
+ for (const report3 of this.listeners.get(repo) ?? []) report3(repo, step);
7590
+ };
7591
+ const work = this.load(gh, repo, pathsOnly, say).finally(() => {
7592
+ this.inflight.delete(repo);
7593
+ this.listeners.delete(repo);
7594
+ });
7564
7595
  this.inflight.set(repo, work);
7565
7596
  return work;
7566
7597
  }
@@ -7581,7 +7612,8 @@ var Snapshots = class {
7581
7612
  if (commit2) snap.sha = commit2;
7582
7613
  void this.kv.put(`blob:${sha}`, text2);
7583
7614
  }
7584
- async load(gh, repo, pathsOnly) {
7615
+ async load(gh, repo, pathsOnly, say) {
7616
+ say("checking for changes");
7585
7617
  const head2 = await gh.request("GET", `repos/${repo}/commits/HEAD`, {
7586
7618
  accept: "application/vnd.github.sha",
7587
7619
  as: "text"
@@ -7593,14 +7625,16 @@ var Snapshots = class {
7593
7625
  const sha = head2.data.trim();
7594
7626
  const before = this.repos.get(repo);
7595
7627
  if (before && before.sha === sha && before.pathsOnly === pathsOnly) {
7628
+ say("up to date");
7596
7629
  return { ok: true, changed: false };
7597
7630
  }
7598
- const next = await this.build(gh, repo, sha, pathsOnly, before);
7631
+ say("reading the file list");
7632
+ const next = await this.build(gh, repo, sha, pathsOnly, say, before);
7599
7633
  if (!next.ok) return { ok: false, changed: false, error: next.error, status: next.status };
7600
7634
  this.repos.set(repo, next.snap);
7601
7635
  return { ok: true, changed: true };
7602
7636
  }
7603
- async build(gh, repo, sha, pathsOnly, before) {
7637
+ async build(gh, repo, sha, pathsOnly, say, before) {
7604
7638
  const [tree, commit2] = await Promise.all([
7605
7639
  gh.request(
7606
7640
  "GET",
@@ -7643,6 +7677,7 @@ var Snapshots = class {
7643
7677
  })
7644
7678
  );
7645
7679
  if (missing2.length > TARBALL_OVER) {
7680
+ say("downloading the repository");
7646
7681
  const tgz = await gh.request("GET", `repos/${repo}/tarball/${sha}`, {
7647
7682
  as: "bytes"
7648
7683
  });
@@ -7657,6 +7692,9 @@ var Snapshots = class {
7657
7692
  void this.kv.put(`blob:${e.sha}`, text2);
7658
7693
  }
7659
7694
  } else {
7695
+ if (missing2.length) {
7696
+ say(`fetching ${missing2.length} changed file${missing2.length === 1 ? "" : "s"}`);
7697
+ }
7660
7698
  await pool2(missing2, 8, async ([, e]) => {
7661
7699
  const res = await gh.request("GET", `repos/${repo}/git/blobs/${e.sha}`, {
7662
7700
  accept: "application/vnd.github.raw",
@@ -7669,7 +7707,7 @@ var Snapshots = class {
7669
7707
  void this.kv.put(`blob:${e.sha}`, text2);
7670
7708
  });
7671
7709
  }
7672
- await this.dates(gh, repo, snap, before);
7710
+ await this.dates(gh, repo, snap, say, before);
7673
7711
  return { ok: true, snap };
7674
7712
  }
7675
7713
  /**
@@ -7679,7 +7717,7 @@ var Snapshots = class {
7679
7717
  * GitHub, in batches, only for the ones that are new or changed. Kept by commit, so a warm
7680
7718
  * start never asks at all.
7681
7719
  */
7682
- async dates(gh, repo, snap, before) {
7720
+ async dates(gh, repo, snap, say, before) {
7683
7721
  const key = `dates:${repo}@${snap.sha}`;
7684
7722
  const kept = await this.kv.get(key);
7685
7723
  if (kept) {
@@ -7694,6 +7732,7 @@ var Snapshots = class {
7694
7732
  else ask.push(path);
7695
7733
  }
7696
7734
  const [owner, name] = repo.split("/");
7735
+ if (ask.length) say("reading file dates");
7697
7736
  for (let i = 0; i < ask.length; i += DATES_PER_QUERY) {
7698
7737
  const batch = ask.slice(i, i + DATES_PER_QUERY);
7699
7738
  const fields = batch.map(
@@ -7779,7 +7818,7 @@ var TTL = 45e3;
7779
7818
  var LABEL_TTL = 3e5;
7780
7819
  function createPortal(opts) {
7781
7820
  const root2 = opts.root ?? `/@roster/${opts.org}`;
7782
- const snapshots = new Snapshots(opts.kv);
7821
+ const snapshots = new Snapshots(opts.kv, void 0, opts.shareInflight ?? true);
7783
7822
  mount(
7784
7823
  root2,
7785
7824
  snapshots.tree((repo) => repo.slice(repo.indexOf("/") + 1))
@@ -7824,19 +7863,24 @@ async function handlePortal(req, portal, actor) {
7824
7863
  }
7825
7864
  async function ready(c, opts = {}) {
7826
7865
  const { portal, actor } = c;
7827
- const ops = await portal.snapshots.refresh(actor.gh, `${portal.org}/${portal.opsName}`, opts);
7866
+ const report3 = opts.report;
7867
+ const ops = await portal.snapshots.refresh(actor.gh, `${portal.org}/${portal.opsName}`, {
7868
+ force: opts.force,
7869
+ report: report3
7870
+ });
7828
7871
  if (!ops.ok) {
7829
7872
  throw new HttpError(
7830
7873
  ops.status === 404 || ops.status === 401 ? ops.status : 502,
7831
7874
  `could not read ${portal.org}/${portal.opsName}: ${ops.error}`
7832
7875
  );
7833
7876
  }
7877
+ report3?.(`${portal.org}/${portal.opsName}`, "ready");
7834
7878
  const org = readOrg(portal.ws.opsDir, parseYaml2);
7835
7879
  const brains = (org.staff ?? []).map((s) => String(s.dir ?? s.handle));
7836
7880
  const products = opts.products ? (org.repos ?? []).map((r) => String(r.name)).filter((n) => n !== portal.opsName && !brains.includes(n)) : [];
7837
7881
  const results = await Promise.all([
7838
7882
  ...brains.map(
7839
- (d) => portal.snapshots.refresh(actor.gh, `${portal.org}/${d}`, { force: opts.force }).then((r) => ({ dir: d, ...r }))
7883
+ (d) => portal.snapshots.refresh(actor.gh, `${portal.org}/${d}`, { force: opts.force, report: report3 }).then((r) => ({ dir: d, ...r }))
7840
7884
  ),
7841
7885
  ...products.map(
7842
7886
  (d) => portal.snapshots.refresh(actor.gh, `${portal.org}/${d}`, { force: opts.force, pathsOnly: true }).then((r) => ({ dir: d, ...r }))
@@ -7864,24 +7908,26 @@ var ROUTES = {
7864
7908
  headers: { "content-type": asset.type, "cache-control": "public, max-age=86400" }
7865
7909
  });
7866
7910
  },
7911
+ /* The whole org. With ?progress=1 it is a stream instead: a line per step as it is read from
7912
+ GitHub, then the org, so a first load can say what it is doing rather than "Loading…". */
7867
7913
  "/api/org": async (c) => {
7868
- const { portal, actor } = c;
7869
- const { org, brains } = await ready(c);
7870
- const byRoot = /* @__PURE__ */ new Map();
7871
- await Promise.all(
7872
- brains.map(async (dir) => {
7873
- const repo = `${portal.org}/${dir}`;
7874
- const snap = portal.snapshots.get(repo);
7875
- if (!snap) return;
7876
- byRoot.set(
7877
- join29(portal.ws.root, dir),
7878
- await loadHistory(actor.gh, portal.kv, repo, snap.sha)
7879
- );
7880
- })
7881
- );
7882
- const data = buildExport(portal.ws, org, parseYaml2, { history: loadedHistory(byRoot) });
7883
- const left = unfinished(portal.ws.opsDir, (org.staff ?? []).length);
7884
- return json({ ...data, unfinished: left });
7914
+ if (c.url.searchParams.get("progress") !== "1") return json(await orgExport(c));
7915
+ const { readable, writable } = new TransformStream();
7916
+ const out = writable.getWriter();
7917
+ const line = (o) => out.write(new TextEncoder().encode(`${JSON.stringify(o)}
7918
+ `));
7919
+ const report3 = (repo, step) => {
7920
+ void line({ repo: repo.slice(repo.indexOf("/") + 1), step });
7921
+ };
7922
+ void orgExport(c, report3).then((org) => line({ org })).catch(
7923
+ (err) => line({ error: String(err?.message ?? err), status: err.status })
7924
+ ).finally(() => out.close());
7925
+ return new Response(readable, {
7926
+ headers: {
7927
+ "content-type": "application/x-ndjson; charset=utf-8",
7928
+ "cache-control": "no-store"
7929
+ }
7930
+ });
7885
7931
  },
7886
7932
  /* Every file in the org layer the portal may write: isWritable's allowlist, nothing more. */
7887
7933
  "/api/orglayer": async (c) => {
@@ -8443,6 +8489,24 @@ ${diff}`,
8443
8489
  return new Response(bytes2, { headers: { "content-type": type, "cache-control": "no-store" } });
8444
8490
  }
8445
8491
  };
8492
+ async function orgExport(c, report3) {
8493
+ const { portal, actor } = c;
8494
+ const { org, brains } = await ready(c, { report: report3 });
8495
+ const byRoot = /* @__PURE__ */ new Map();
8496
+ await Promise.all(
8497
+ brains.map(async (dir) => {
8498
+ const repo = `${portal.org}/${dir}`;
8499
+ const snap = portal.snapshots.get(repo);
8500
+ if (!snap) return;
8501
+ report3?.(repo, "reading recent history");
8502
+ byRoot.set(join29(portal.ws.root, dir), await loadHistory(actor.gh, portal.kv, repo, snap.sha));
8503
+ report3?.(repo, "ready");
8504
+ })
8505
+ );
8506
+ const data = buildExport(portal.ws, org, parseYaml2, { history: loadedHistory(byRoot) });
8507
+ const left = unfinished(portal.ws.opsDir, (org.staff ?? []).length);
8508
+ return { ...data, unfinished: left };
8509
+ }
8446
8510
  function knownRepos(portal) {
8447
8511
  const org = readOrg(portal.ws.opsDir, parseYaml2);
8448
8512
  return (org.repos ?? []).map((r) => ({
@@ -9010,7 +9074,17 @@ async function portalCommand(argv) {
9010
9074
  return;
9011
9075
  }
9012
9076
  res.writeHead(answer.status, Object.fromEntries(answer.headers));
9013
- res.end(Buffer.from(await answer.arrayBuffer()));
9077
+ if (!answer.body) {
9078
+ res.end();
9079
+ return;
9080
+ }
9081
+ const reader = answer.body.getReader();
9082
+ for (; ; ) {
9083
+ const { done, value } = await reader.read();
9084
+ if (done) break;
9085
+ res.write(value);
9086
+ }
9087
+ res.end();
9014
9088
  };
9015
9089
  const writeAllowed = (req) => {
9016
9090
  if (req.method !== "POST") return false;
@@ -310,3 +310,16 @@ dialog.sheet.side .sidebar-x .iconbtn{margin-top:-4px}
310
310
  .spinner{width:22px;height:22px;border-radius:50%;border:2px solid var(--line);border-top-color:var(--accent);
311
311
  animation:spin .7s linear infinite}
312
312
  @keyframes spin{to{transform:rotate(360deg)}}
313
+
314
+ /* The first load, while the org is read from GitHub: a row per repository. */
315
+ .loadlist{display:flex;flex-direction:column;gap:2px;max-width:520px}
316
+ .loadrow{display:grid;grid-template-columns:18px 140px 1fr;align-items:center;gap:10px;padding:7px 0;
317
+ font-size:13.5px;border-bottom:1px solid var(--line)}
318
+ .loadrow b{font-weight:500}
319
+ .loadrow .meta{color:var(--ink-dim)}
320
+ .loadmark{width:10px;height:10px;border-radius:50%;border:2px solid var(--accent-dim);
321
+ border-top-color:var(--accent);animation:loadspin .8s linear infinite}
322
+ .loadrow.done .loadmark{animation:none;border-color:var(--accent);background:var(--accent)}
323
+ .loadrow.done .meta{color:var(--ink-faint)}
324
+ @keyframes loadspin{to{transform:rotate(360deg)}}
325
+ @media (prefers-reduced-motion:reduce){.loadmark{animation:none}}
@@ -14,6 +14,36 @@ export const getSession = () =>
14
14
  .catch(() => null);
15
15
 
16
16
  export const getOrg = () => json("/api/org");
17
+
18
+ /**
19
+ * The org, for the first load, saying what it is doing as it goes: `onStep(repo, step)` for each
20
+ * line the server streams, then the org itself. Anything that is not a stream (the setup answer,
21
+ * an older server) is read as plain JSON, so this is never worse than getOrg.
22
+ */
23
+ export async function getOrgWithProgress(onStep) {
24
+ const res = await fetch("/api/org?progress=1", { cache: "no-store" });
25
+ if (res.status === 401) location.reload();
26
+ const type = String(res.headers?.get?.("content-type") ?? "");
27
+ if (!type.includes("ndjson") || !res.body?.pipeThrough) return res.json();
28
+ const reader = res.body.pipeThrough(new TextDecoderStream()).getReader();
29
+ let buf = "";
30
+ for (;;) {
31
+ const { done, value } = await reader.read();
32
+ if (done) break;
33
+ buf += value;
34
+ let nl;
35
+ while ((nl = buf.indexOf("\n")) >= 0) {
36
+ const line = buf.slice(0, nl).trim();
37
+ buf = buf.slice(nl + 1);
38
+ if (!line) continue;
39
+ const msg = JSON.parse(line);
40
+ if (msg.org) return msg.org;
41
+ if (msg.error) throw new Error(msg.error);
42
+ onStep(msg.repo, msg.step);
43
+ }
44
+ }
45
+ throw new Error("The org did not arrive. Reload to try again.");
46
+ }
17
47
  export const getDocs = () => json("/api/docs");
18
48
  export const getInbox = (force) => json("/api/inbox" + (force ? "?refresh=1" : ""));
19
49
  export const getSync = () => json("/api/sync").catch(() => null);
@@ -1,6 +1,6 @@
1
1
  /* The shell: boot, the sidebar, and dispatching a view into <main>. */
2
2
 
3
- import { getOrg, getSession } from "./api.js";
3
+ import { getOrgWithProgress, getSession } from "./api.js";
4
4
  import { $, el, store } from "./dom.js";
5
5
  import { icon, iconHTML, staffIcon } from "./icons.js";
6
6
  import { installLightbox } from "./lightbox.js";
@@ -96,7 +96,7 @@ export async function boot() {
96
96
  /* Before there is a tenant the server answers every data route with `mode: setup`. That is
97
97
  not an error state: it is the first thing a new user ever sees, and the whole page becomes
98
98
  the thing that fixes it. */
99
- const [first, session] = await Promise.all([getOrg(), getSession()]);
99
+ const [first, session] = await Promise.all([loadWithProgress(), getSession()]);
100
100
  S.hosted = session?.hosted ? session : null;
101
101
  if (S.hosted) showSignedIn(S.hosted.login);
102
102
  if (first && first.mode === "setup") {
@@ -409,6 +409,41 @@ function render() {
409
409
  (SCREEN[S.view] ?? viewBrain)(m);
410
410
  }
411
411
 
412
+ /* ------------------------------- first load -------------------------------- */
413
+
414
+ /**
415
+ * The first read of the org, with what it is doing on screen. A cold start reads every
416
+ * repository from GitHub, which can take a while, and "Loading…" for fifteen seconds reads as
417
+ * broken. One row per repository, saying which step it is on, and a tick when it is ready.
418
+ */
419
+ async function loadWithProgress() {
420
+ const main = $("#main");
421
+ const rows = new Map();
422
+ const list = el("div", { className: "loadlist" });
423
+ const head = el("p", { className: "sub", textContent: "Reading the org from GitHub…" });
424
+ // Only once something is actually slow: a warm load is over before this would be read.
425
+ const show = setTimeout(() => main.replaceChildren(head, list), 400);
426
+ try {
427
+ return await getOrgWithProgress((repo, step) => {
428
+ let row = rows.get(repo);
429
+ if (!row) {
430
+ row = el("div", { className: "loadrow" }, [
431
+ el("span", { className: "loadmark" }),
432
+ el("b", { textContent: repo }),
433
+ el("span", { className: "meta" }),
434
+ ]);
435
+ rows.set(repo, row);
436
+ list.append(row);
437
+ }
438
+ const ready = step === "ready" || step === "up to date";
439
+ row.classList.toggle("done", ready);
440
+ row.lastChild.textContent = ready ? "ready" : step;
441
+ });
442
+ } finally {
443
+ clearTimeout(show);
444
+ }
445
+ }
446
+
412
447
  /* -------------------------------- signed in -------------------------------- */
413
448
 
414
449
  /**
@@ -2218,7 +2218,7 @@ var TIMELINE_PR = `
2218
2218
  ... on ReadyForReviewEvent { actor { login } createdAt }
2219
2219
  ... on ReviewRequestedEvent {
2220
2220
  actor { login } createdAt
2221
- requestedReviewer { ... on User { login } ... on Team { name } }
2221
+ requestedReviewer { __typename ... on User { login } ... on Bot { login } }
2222
2222
  }
2223
2223
  `;
2224
2224
  var CLOSED_FIRST = 30;
@@ -2463,7 +2463,9 @@ function event(e) {
2463
2463
  type: "review-requested",
2464
2464
  actor: who,
2465
2465
  createdAt: at,
2466
- assignee: e.requestedReviewer?.login ?? e.requestedReviewer?.name ?? ""
2466
+ /* A team's name needs the read:org scope, which a portal signed in with `repo` alone does
2467
+ not have, and asking for it fails the whole thread. Its type needs nothing. */
2468
+ assignee: e.requestedReviewer?.login ?? (e.requestedReviewer?.__typename === "Team" ? "a team" : "")
2467
2469
  };
2468
2470
  case "LabeledEvent":
2469
2471
  return { type: "labeled", actor: who, createdAt: at, label: e.label?.name ?? "" };
@@ -2992,15 +2994,19 @@ function isTextPath(path, size2) {
2992
2994
  return size2 <= MAX_TEXT && !BINARY.test(path);
2993
2995
  }
2994
2996
  var Snapshots = class {
2995
- constructor(kv, freshMs = 4e3) {
2997
+ constructor(kv, freshMs = 4e3, shareInflight = true) {
2996
2998
  this.kv = kv;
2997
2999
  this.freshMs = freshMs;
3000
+ this.shareInflight = shareInflight;
2998
3001
  }
2999
3002
  kv;
3000
3003
  freshMs;
3004
+ shareInflight;
3001
3005
  repos = /* @__PURE__ */ new Map();
3002
3006
  inflight = /* @__PURE__ */ new Map();
3003
3007
  checked = /* @__PURE__ */ new Map();
3008
+ /** Who wants to hear how a refresh in flight is going. Every caller that joined it. */
3009
+ listeners = /* @__PURE__ */ new Map();
3004
3010
  get(repo) {
3005
3011
  return this.repos.get(repo);
3006
3012
  }
@@ -3015,11 +3021,23 @@ var Snapshots = class {
3015
3021
  if (!opts.force && this.repos.has(repo) && Date.now() - last < this.freshMs) {
3016
3022
  return Promise.resolve({ ok: true, changed: false });
3017
3023
  }
3024
+ const pathsOnly = opts.pathsOnly === true;
3025
+ if (!this.shareInflight) {
3026
+ return this.load(gh, repo, pathsOnly, (step) => opts.report?.(repo, step));
3027
+ }
3028
+ if (opts.report) {
3029
+ if (!this.listeners.has(repo)) this.listeners.set(repo, /* @__PURE__ */ new Set());
3030
+ this.listeners.get(repo).add(opts.report);
3031
+ }
3018
3032
  const running = this.inflight.get(repo);
3019
3033
  if (running) return running;
3020
- const work = this.load(gh, repo, opts.pathsOnly === true).finally(
3021
- () => this.inflight.delete(repo)
3022
- );
3034
+ const say = (step) => {
3035
+ for (const report of this.listeners.get(repo) ?? []) report(repo, step);
3036
+ };
3037
+ const work = this.load(gh, repo, pathsOnly, say).finally(() => {
3038
+ this.inflight.delete(repo);
3039
+ this.listeners.delete(repo);
3040
+ });
3023
3041
  this.inflight.set(repo, work);
3024
3042
  return work;
3025
3043
  }
@@ -3040,7 +3058,8 @@ var Snapshots = class {
3040
3058
  if (commit2) snap.sha = commit2;
3041
3059
  void this.kv.put(`blob:${sha}`, text2);
3042
3060
  }
3043
- async load(gh, repo, pathsOnly) {
3061
+ async load(gh, repo, pathsOnly, say) {
3062
+ say("checking for changes");
3044
3063
  const head2 = await gh.request("GET", `repos/${repo}/commits/HEAD`, {
3045
3064
  accept: "application/vnd.github.sha",
3046
3065
  as: "text"
@@ -3052,14 +3071,16 @@ var Snapshots = class {
3052
3071
  const sha = head2.data.trim();
3053
3072
  const before = this.repos.get(repo);
3054
3073
  if (before && before.sha === sha && before.pathsOnly === pathsOnly) {
3074
+ say("up to date");
3055
3075
  return { ok: true, changed: false };
3056
3076
  }
3057
- const next = await this.build(gh, repo, sha, pathsOnly, before);
3077
+ say("reading the file list");
3078
+ const next = await this.build(gh, repo, sha, pathsOnly, say, before);
3058
3079
  if (!next.ok) return { ok: false, changed: false, error: next.error, status: next.status };
3059
3080
  this.repos.set(repo, next.snap);
3060
3081
  return { ok: true, changed: true };
3061
3082
  }
3062
- async build(gh, repo, sha, pathsOnly, before) {
3083
+ async build(gh, repo, sha, pathsOnly, say, before) {
3063
3084
  const [tree, commit2] = await Promise.all([
3064
3085
  gh.request(
3065
3086
  "GET",
@@ -3102,6 +3123,7 @@ var Snapshots = class {
3102
3123
  })
3103
3124
  );
3104
3125
  if (missing2.length > TARBALL_OVER) {
3126
+ say("downloading the repository");
3105
3127
  const tgz = await gh.request("GET", `repos/${repo}/tarball/${sha}`, {
3106
3128
  as: "bytes"
3107
3129
  });
@@ -3116,6 +3138,9 @@ var Snapshots = class {
3116
3138
  void this.kv.put(`blob:${e.sha}`, text2);
3117
3139
  }
3118
3140
  } else {
3141
+ if (missing2.length) {
3142
+ say(`fetching ${missing2.length} changed file${missing2.length === 1 ? "" : "s"}`);
3143
+ }
3119
3144
  await pool2(missing2, 8, async ([, e]) => {
3120
3145
  const res = await gh.request("GET", `repos/${repo}/git/blobs/${e.sha}`, {
3121
3146
  accept: "application/vnd.github.raw",
@@ -3128,7 +3153,7 @@ var Snapshots = class {
3128
3153
  void this.kv.put(`blob:${e.sha}`, text2);
3129
3154
  });
3130
3155
  }
3131
- await this.dates(gh, repo, snap, before);
3156
+ await this.dates(gh, repo, snap, say, before);
3132
3157
  return { ok: true, snap };
3133
3158
  }
3134
3159
  /**
@@ -3138,7 +3163,7 @@ var Snapshots = class {
3138
3163
  * GitHub, in batches, only for the ones that are new or changed. Kept by commit, so a warm
3139
3164
  * start never asks at all.
3140
3165
  */
3141
- async dates(gh, repo, snap, before) {
3166
+ async dates(gh, repo, snap, say, before) {
3142
3167
  const key = `dates:${repo}@${snap.sha}`;
3143
3168
  const kept = await this.kv.get(key);
3144
3169
  if (kept) {
@@ -3153,6 +3178,7 @@ var Snapshots = class {
3153
3178
  else ask.push(path);
3154
3179
  }
3155
3180
  const [owner, name] = repo.split("/");
3181
+ if (ask.length) say("reading file dates");
3156
3182
  for (let i = 0; i < ask.length; i += DATES_PER_QUERY) {
3157
3183
  const batch = ask.slice(i, i + DATES_PER_QUERY);
3158
3184
  const fields = batch.map(
@@ -3238,7 +3264,7 @@ var TTL = 45e3;
3238
3264
  var LABEL_TTL = 3e5;
3239
3265
  function createPortal(opts) {
3240
3266
  const root2 = opts.root ?? `/@roster/${opts.org}`;
3241
- const snapshots = new Snapshots(opts.kv);
3267
+ const snapshots = new Snapshots(opts.kv, void 0, opts.shareInflight ?? true);
3242
3268
  mount(
3243
3269
  root2,
3244
3270
  snapshots.tree((repo) => repo.slice(repo.indexOf("/") + 1))
@@ -3283,19 +3309,24 @@ async function handlePortal(req, portal2, actor) {
3283
3309
  }
3284
3310
  async function ready(c, opts = {}) {
3285
3311
  const { portal: portal2, actor } = c;
3286
- const ops = await portal2.snapshots.refresh(actor.gh, `${portal2.org}/${portal2.opsName}`, opts);
3312
+ const report = opts.report;
3313
+ const ops = await portal2.snapshots.refresh(actor.gh, `${portal2.org}/${portal2.opsName}`, {
3314
+ force: opts.force,
3315
+ report
3316
+ });
3287
3317
  if (!ops.ok) {
3288
3318
  throw new HttpError(
3289
3319
  ops.status === 404 || ops.status === 401 ? ops.status : 502,
3290
3320
  `could not read ${portal2.org}/${portal2.opsName}: ${ops.error}`
3291
3321
  );
3292
3322
  }
3323
+ report?.(`${portal2.org}/${portal2.opsName}`, "ready");
3293
3324
  const org = readOrg(portal2.ws.opsDir, parseYaml2);
3294
3325
  const brains = (org.staff ?? []).map((s) => String(s.dir ?? s.handle));
3295
3326
  const products = opts.products ? (org.repos ?? []).map((r) => String(r.name)).filter((n) => n !== portal2.opsName && !brains.includes(n)) : [];
3296
3327
  const results = await Promise.all([
3297
3328
  ...brains.map(
3298
- (d) => portal2.snapshots.refresh(actor.gh, `${portal2.org}/${d}`, { force: opts.force }).then((r) => ({ dir: d, ...r }))
3329
+ (d) => portal2.snapshots.refresh(actor.gh, `${portal2.org}/${d}`, { force: opts.force, report }).then((r) => ({ dir: d, ...r }))
3299
3330
  ),
3300
3331
  ...products.map(
3301
3332
  (d) => portal2.snapshots.refresh(actor.gh, `${portal2.org}/${d}`, { force: opts.force, pathsOnly: true }).then((r) => ({ dir: d, ...r }))
@@ -3327,24 +3358,26 @@ var ROUTES = {
3327
3358
  headers: { "content-type": asset.type, "cache-control": "public, max-age=86400" }
3328
3359
  });
3329
3360
  },
3361
+ /* The whole org. With ?progress=1 it is a stream instead: a line per step as it is read from
3362
+ GitHub, then the org, so a first load can say what it is doing rather than "Loading…". */
3330
3363
  "/api/org": async (c) => {
3331
- const { portal: portal2, actor } = c;
3332
- const { org, brains } = await ready(c);
3333
- const byRoot = /* @__PURE__ */ new Map();
3334
- await Promise.all(
3335
- brains.map(async (dir) => {
3336
- const repo = `${portal2.org}/${dir}`;
3337
- const snap = portal2.snapshots.get(repo);
3338
- if (!snap) return;
3339
- byRoot.set(
3340
- join17(portal2.ws.root, dir),
3341
- await loadHistory(actor.gh, portal2.kv, repo, snap.sha)
3342
- );
3343
- })
3344
- );
3345
- const data = buildExport(portal2.ws, org, parseYaml2, { history: loadedHistory(byRoot) });
3346
- const left = unfinished(portal2.ws.opsDir, (org.staff ?? []).length);
3347
- return json({ ...data, unfinished: left });
3364
+ if (c.url.searchParams.get("progress") !== "1") return json(await orgExport(c));
3365
+ const { readable, writable } = new TransformStream();
3366
+ const out = writable.getWriter();
3367
+ const line = (o) => out.write(new TextEncoder().encode(`${JSON.stringify(o)}
3368
+ `));
3369
+ const report = (repo, step) => {
3370
+ void line({ repo: repo.slice(repo.indexOf("/") + 1), step });
3371
+ };
3372
+ void orgExport(c, report).then((org) => line({ org })).catch(
3373
+ (err) => line({ error: String(err?.message ?? err), status: err.status })
3374
+ ).finally(() => out.close());
3375
+ return new Response(readable, {
3376
+ headers: {
3377
+ "content-type": "application/x-ndjson; charset=utf-8",
3378
+ "cache-control": "no-store"
3379
+ }
3380
+ });
3348
3381
  },
3349
3382
  /* Every file in the org layer the portal may write: isWritable's allowlist, nothing more. */
3350
3383
  "/api/orglayer": async (c) => {
@@ -3906,6 +3939,24 @@ ${diff}`,
3906
3939
  return new Response(bytes, { headers: { "content-type": type, "cache-control": "no-store" } });
3907
3940
  }
3908
3941
  };
3942
+ async function orgExport(c, report) {
3943
+ const { portal: portal2, actor } = c;
3944
+ const { org, brains } = await ready(c, { report });
3945
+ const byRoot = /* @__PURE__ */ new Map();
3946
+ await Promise.all(
3947
+ brains.map(async (dir) => {
3948
+ const repo = `${portal2.org}/${dir}`;
3949
+ const snap = portal2.snapshots.get(repo);
3950
+ if (!snap) return;
3951
+ report?.(repo, "reading recent history");
3952
+ byRoot.set(join17(portal2.ws.root, dir), await loadHistory(actor.gh, portal2.kv, repo, snap.sha));
3953
+ report?.(repo, "ready");
3954
+ })
3955
+ );
3956
+ const data = buildExport(portal2.ws, org, parseYaml2, { history: loadedHistory(byRoot) });
3957
+ const left = unfinished(portal2.ws.opsDir, (org.staff ?? []).length);
3958
+ return { ...data, unfinished: left };
3959
+ }
3909
3960
  function knownRepos(portal2) {
3910
3961
  const org = readOrg(portal2.ws.opsDir, parseYaml2);
3911
3962
  return (org.repos ?? []).map((r) => ({
@@ -4353,7 +4404,13 @@ function boot(env) {
4353
4404
  useFrameworkRoot(FRAMEWORK);
4354
4405
  const kv = cloudflareKV(env.CACHE);
4355
4406
  useRecordStore(recordStore(kv));
4356
- portal = createPortal({ org: env.ROSTER_ORG, opsName: env.ROSTER_OPS_REPO ?? "roster-ops", kv });
4407
+ portal = createPortal({
4408
+ org: env.ROSTER_ORG,
4409
+ opsName: env.ROSTER_OPS_REPO ?? "roster-ops",
4410
+ kv,
4411
+ // A Worker request that awaits a fetch another request started never finishes.
4412
+ shareInflight: false
4413
+ });
4357
4414
  return portal;
4358
4415
  }
4359
4416
  function clientFor(token) {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@nanocollective/roster",
3
- "version": "0.1.0-beta.2",
3
+ "version": "0.1.0-beta.4",
4
4
  "description": "An agent-run org, powered by GitHub. Scaffold AI staff members whose brain is a repo.",
5
5
  "type": "module",
6
6
  "bin": {
@@ -310,3 +310,16 @@ dialog.sheet.side .sidebar-x .iconbtn{margin-top:-4px}
310
310
  .spinner{width:22px;height:22px;border-radius:50%;border:2px solid var(--line);border-top-color:var(--accent);
311
311
  animation:spin .7s linear infinite}
312
312
  @keyframes spin{to{transform:rotate(360deg)}}
313
+
314
+ /* The first load, while the org is read from GitHub: a row per repository. */
315
+ .loadlist{display:flex;flex-direction:column;gap:2px;max-width:520px}
316
+ .loadrow{display:grid;grid-template-columns:18px 140px 1fr;align-items:center;gap:10px;padding:7px 0;
317
+ font-size:13.5px;border-bottom:1px solid var(--line)}
318
+ .loadrow b{font-weight:500}
319
+ .loadrow .meta{color:var(--ink-dim)}
320
+ .loadmark{width:10px;height:10px;border-radius:50%;border:2px solid var(--accent-dim);
321
+ border-top-color:var(--accent);animation:loadspin .8s linear infinite}
322
+ .loadrow.done .loadmark{animation:none;border-color:var(--accent);background:var(--accent)}
323
+ .loadrow.done .meta{color:var(--ink-faint)}
324
+ @keyframes loadspin{to{transform:rotate(360deg)}}
325
+ @media (prefers-reduced-motion:reduce){.loadmark{animation:none}}
@@ -14,6 +14,36 @@ export const getSession = () =>
14
14
  .catch(() => null);
15
15
 
16
16
  export const getOrg = () => json("/api/org");
17
+
18
+ /**
19
+ * The org, for the first load, saying what it is doing as it goes: `onStep(repo, step)` for each
20
+ * line the server streams, then the org itself. Anything that is not a stream (the setup answer,
21
+ * an older server) is read as plain JSON, so this is never worse than getOrg.
22
+ */
23
+ export async function getOrgWithProgress(onStep) {
24
+ const res = await fetch("/api/org?progress=1", { cache: "no-store" });
25
+ if (res.status === 401) location.reload();
26
+ const type = String(res.headers?.get?.("content-type") ?? "");
27
+ if (!type.includes("ndjson") || !res.body?.pipeThrough) return res.json();
28
+ const reader = res.body.pipeThrough(new TextDecoderStream()).getReader();
29
+ let buf = "";
30
+ for (;;) {
31
+ const { done, value } = await reader.read();
32
+ if (done) break;
33
+ buf += value;
34
+ let nl;
35
+ while ((nl = buf.indexOf("\n")) >= 0) {
36
+ const line = buf.slice(0, nl).trim();
37
+ buf = buf.slice(nl + 1);
38
+ if (!line) continue;
39
+ const msg = JSON.parse(line);
40
+ if (msg.org) return msg.org;
41
+ if (msg.error) throw new Error(msg.error);
42
+ onStep(msg.repo, msg.step);
43
+ }
44
+ }
45
+ throw new Error("The org did not arrive. Reload to try again.");
46
+ }
17
47
  export const getDocs = () => json("/api/docs");
18
48
  export const getInbox = (force) => json("/api/inbox" + (force ? "?refresh=1" : ""));
19
49
  export const getSync = () => json("/api/sync").catch(() => null);
@@ -1,6 +1,6 @@
1
1
  /* The shell: boot, the sidebar, and dispatching a view into <main>. */
2
2
 
3
- import { getOrg, getSession } from "./api.js";
3
+ import { getOrgWithProgress, getSession } from "./api.js";
4
4
  import { $, el, store } from "./dom.js";
5
5
  import { icon, iconHTML, staffIcon } from "./icons.js";
6
6
  import { installLightbox } from "./lightbox.js";
@@ -96,7 +96,7 @@ export async function boot() {
96
96
  /* Before there is a tenant the server answers every data route with `mode: setup`. That is
97
97
  not an error state: it is the first thing a new user ever sees, and the whole page becomes
98
98
  the thing that fixes it. */
99
- const [first, session] = await Promise.all([getOrg(), getSession()]);
99
+ const [first, session] = await Promise.all([loadWithProgress(), getSession()]);
100
100
  S.hosted = session?.hosted ? session : null;
101
101
  if (S.hosted) showSignedIn(S.hosted.login);
102
102
  if (first && first.mode === "setup") {
@@ -409,6 +409,41 @@ function render() {
409
409
  (SCREEN[S.view] ?? viewBrain)(m);
410
410
  }
411
411
 
412
+ /* ------------------------------- first load -------------------------------- */
413
+
414
+ /**
415
+ * The first read of the org, with what it is doing on screen. A cold start reads every
416
+ * repository from GitHub, which can take a while, and "Loading…" for fifteen seconds reads as
417
+ * broken. One row per repository, saying which step it is on, and a tick when it is ready.
418
+ */
419
+ async function loadWithProgress() {
420
+ const main = $("#main");
421
+ const rows = new Map();
422
+ const list = el("div", { className: "loadlist" });
423
+ const head = el("p", { className: "sub", textContent: "Reading the org from GitHub…" });
424
+ // Only once something is actually slow: a warm load is over before this would be read.
425
+ const show = setTimeout(() => main.replaceChildren(head, list), 400);
426
+ try {
427
+ return await getOrgWithProgress((repo, step) => {
428
+ let row = rows.get(repo);
429
+ if (!row) {
430
+ row = el("div", { className: "loadrow" }, [
431
+ el("span", { className: "loadmark" }),
432
+ el("b", { textContent: repo }),
433
+ el("span", { className: "meta" }),
434
+ ]);
435
+ rows.set(repo, row);
436
+ list.append(row);
437
+ }
438
+ const ready = step === "ready" || step === "up to date";
439
+ row.classList.toggle("done", ready);
440
+ row.lastChild.textContent = ready ? "ready" : step;
441
+ });
442
+ } finally {
443
+ clearTimeout(show);
444
+ }
445
+ }
446
+
412
447
  /* -------------------------------- signed in -------------------------------- */
413
448
 
414
449
  /**