@nanocollective/roster 0.1.0-beta.1 → 0.1.0-beta.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -47,6 +47,7 @@ roster run cto # one run now, followed to the end
47
47
  roster doctor # is any of this actually wired up
48
48
  roster fix # every finding, as one brief for a coding agent
49
49
  roster portal # read every brain, and the docs, locally
50
+ roster portal deploy # the same portal on your own URL, behind Sign in with GitHub
50
51
  roster prompt cto # the exact prompt a run will be sent
51
52
  roster upgrade # take framework changes without losing your edits
52
53
  roster retire cto # let someone go, with their brain kept
package/dist/cli.js CHANGED
@@ -2913,7 +2913,20 @@ function workerBundle() {
2913
2913
  }
2914
2914
  return null;
2915
2915
  }
2916
- function wrangler(args) {
2916
+ async function wrangler(args) {
2917
+ let code = 1;
2918
+ for (let attempt = 1; attempt <= 3; attempt++) {
2919
+ code = await wranglerOnce(args);
2920
+ if (code === 0) return 0;
2921
+ if (attempt < 3) {
2922
+ process.stdout.write(` wrangler failed; trying again (${attempt + 1} of 3)\u2026
2923
+ `);
2924
+ await new Promise((r) => setTimeout(r, 15e3 * attempt));
2925
+ }
2926
+ }
2927
+ return code;
2928
+ }
2929
+ function wranglerOnce(args) {
2917
2930
  return new Promise((resolve8) => {
2918
2931
  const child = spawn("npx", ["--yes", WRANGLER, ...args], { stdio: "inherit" });
2919
2932
  child.on("close", (code) => resolve8(code ?? 1));
@@ -3038,11 +3051,22 @@ function githubClient(token, opts = {}) {
3038
3051
  body2 = JSON.stringify(o.body);
3039
3052
  h["content-type"] = "application/json";
3040
3053
  }
3054
+ const reads = method === "GET" || o.retry === true;
3041
3055
  let res;
3042
- try {
3043
- res = await doFetch(url, { method, headers: h, body: body2 });
3044
- } catch (err) {
3045
- return { ok: false, error: `GitHub could not be reached: ${err.message}` };
3056
+ for (let attempt = 0; ; attempt++) {
3057
+ try {
3058
+ res = await doFetch(url, { method, headers: h, body: body2 });
3059
+ } catch (err) {
3060
+ if (reads && attempt === 0) continue;
3061
+ return { ok: false, error: `GitHub could not be reached: ${err.message}` };
3062
+ }
3063
+ if (reads && attempt === 0 && [502, 503, 504].includes(res.status)) {
3064
+ await res.body?.cancel().catch(() => {
3065
+ });
3066
+ await new Promise((r) => setTimeout(r, 400));
3067
+ continue;
3068
+ }
3069
+ break;
3046
3070
  }
3047
3071
  if (res.status === 304 && kept) {
3048
3072
  seen.delete(key);
@@ -3072,7 +3096,8 @@ function githubClient(token, opts = {}) {
3072
3096
  const res = await request(
3073
3097
  "POST",
3074
3098
  "graphql",
3075
- { body: { query: query2, variables: vars } }
3099
+ // A query reads; a mutation writes, and is never asked twice.
3100
+ { body: { query: query2, variables: vars }, retry: !/^\s*mutation\b/.test(query2) }
3076
3101
  );
3077
3102
  if (!res.ok) return { ok: false, status: res.status, error: res.error };
3078
3103
  const errors = res.data?.errors ?? [];
@@ -6910,7 +6935,7 @@ var TIMELINE_PR = `
6910
6935
  ... on ReadyForReviewEvent { actor { login } createdAt }
6911
6936
  ... on ReviewRequestedEvent {
6912
6937
  actor { login } createdAt
6913
- requestedReviewer { ... on User { login } ... on Team { name } }
6938
+ requestedReviewer { __typename ... on User { login } ... on Bot { login } }
6914
6939
  }
6915
6940
  `;
6916
6941
  var CLOSED_FIRST = 30;
@@ -7155,7 +7180,9 @@ function event(e) {
7155
7180
  type: "review-requested",
7156
7181
  actor: who,
7157
7182
  createdAt: at,
7158
- assignee: e.requestedReviewer?.login ?? e.requestedReviewer?.name ?? ""
7183
+ /* A team's name needs the read:org scope, which a portal signed in with `repo` alone does
7184
+ not have, and asking for it fails the whole thread. Its type needs nothing. */
7185
+ assignee: e.requestedReviewer?.login ?? (e.requestedReviewer?.__typename === "Team" ? "a team" : "")
7159
7186
  };
7160
7187
  case "LabeledEvent":
7161
7188
  return { type: "labeled", actor: who, createdAt: at, label: e.label?.name ?? "" };
@@ -7530,6 +7557,8 @@ var Snapshots = class {
7530
7557
  repos = /* @__PURE__ */ new Map();
7531
7558
  inflight = /* @__PURE__ */ new Map();
7532
7559
  checked = /* @__PURE__ */ new Map();
7560
+ /** Who wants to hear how a refresh in flight is going. Every caller that joined it. */
7561
+ listeners = /* @__PURE__ */ new Map();
7533
7562
  get(repo) {
7534
7563
  return this.repos.get(repo);
7535
7564
  }
@@ -7544,11 +7573,16 @@ var Snapshots = class {
7544
7573
  if (!opts.force && this.repos.has(repo) && Date.now() - last < this.freshMs) {
7545
7574
  return Promise.resolve({ ok: true, changed: false });
7546
7575
  }
7576
+ if (opts.report) {
7577
+ if (!this.listeners.has(repo)) this.listeners.set(repo, /* @__PURE__ */ new Set());
7578
+ this.listeners.get(repo).add(opts.report);
7579
+ }
7547
7580
  const running = this.inflight.get(repo);
7548
7581
  if (running) return running;
7549
- const work = this.load(gh, repo, opts.pathsOnly === true).finally(
7550
- () => this.inflight.delete(repo)
7551
- );
7582
+ const work = this.load(gh, repo, opts.pathsOnly === true).finally(() => {
7583
+ this.inflight.delete(repo);
7584
+ this.listeners.delete(repo);
7585
+ });
7552
7586
  this.inflight.set(repo, work);
7553
7587
  return work;
7554
7588
  }
@@ -7569,7 +7603,11 @@ var Snapshots = class {
7569
7603
  if (commit2) snap.sha = commit2;
7570
7604
  void this.kv.put(`blob:${sha}`, text2);
7571
7605
  }
7606
+ say(repo, step) {
7607
+ for (const report3 of this.listeners.get(repo) ?? []) report3(repo, step);
7608
+ }
7572
7609
  async load(gh, repo, pathsOnly) {
7610
+ this.say(repo, "checking for changes");
7573
7611
  const head2 = await gh.request("GET", `repos/${repo}/commits/HEAD`, {
7574
7612
  accept: "application/vnd.github.sha",
7575
7613
  as: "text"
@@ -7581,8 +7619,10 @@ var Snapshots = class {
7581
7619
  const sha = head2.data.trim();
7582
7620
  const before = this.repos.get(repo);
7583
7621
  if (before && before.sha === sha && before.pathsOnly === pathsOnly) {
7622
+ this.say(repo, "up to date");
7584
7623
  return { ok: true, changed: false };
7585
7624
  }
7625
+ this.say(repo, "reading the file list");
7586
7626
  const next = await this.build(gh, repo, sha, pathsOnly, before);
7587
7627
  if (!next.ok) return { ok: false, changed: false, error: next.error, status: next.status };
7588
7628
  this.repos.set(repo, next.snap);
@@ -7631,6 +7671,7 @@ var Snapshots = class {
7631
7671
  })
7632
7672
  );
7633
7673
  if (missing2.length > TARBALL_OVER) {
7674
+ this.say(repo, "downloading the repository");
7634
7675
  const tgz = await gh.request("GET", `repos/${repo}/tarball/${sha}`, {
7635
7676
  as: "bytes"
7636
7677
  });
@@ -7645,6 +7686,9 @@ var Snapshots = class {
7645
7686
  void this.kv.put(`blob:${e.sha}`, text2);
7646
7687
  }
7647
7688
  } else {
7689
+ if (missing2.length) {
7690
+ this.say(repo, `fetching ${missing2.length} changed file${missing2.length === 1 ? "" : "s"}`);
7691
+ }
7648
7692
  await pool2(missing2, 8, async ([, e]) => {
7649
7693
  const res = await gh.request("GET", `repos/${repo}/git/blobs/${e.sha}`, {
7650
7694
  accept: "application/vnd.github.raw",
@@ -7682,6 +7726,7 @@ var Snapshots = class {
7682
7726
  else ask.push(path);
7683
7727
  }
7684
7728
  const [owner, name] = repo.split("/");
7729
+ if (ask.length) this.say(repo, "reading file dates");
7685
7730
  for (let i = 0; i < ask.length; i += DATES_PER_QUERY) {
7686
7731
  const batch = ask.slice(i, i + DATES_PER_QUERY);
7687
7732
  const fields = batch.map(
@@ -7812,19 +7857,24 @@ async function handlePortal(req, portal, actor) {
7812
7857
  }
7813
7858
  async function ready(c, opts = {}) {
7814
7859
  const { portal, actor } = c;
7815
- const ops = await portal.snapshots.refresh(actor.gh, `${portal.org}/${portal.opsName}`, opts);
7860
+ const report3 = opts.report;
7861
+ const ops = await portal.snapshots.refresh(actor.gh, `${portal.org}/${portal.opsName}`, {
7862
+ force: opts.force,
7863
+ report: report3
7864
+ });
7816
7865
  if (!ops.ok) {
7817
7866
  throw new HttpError(
7818
7867
  ops.status === 404 || ops.status === 401 ? ops.status : 502,
7819
7868
  `could not read ${portal.org}/${portal.opsName}: ${ops.error}`
7820
7869
  );
7821
7870
  }
7871
+ report3?.(`${portal.org}/${portal.opsName}`, "ready");
7822
7872
  const org = readOrg(portal.ws.opsDir, parseYaml2);
7823
7873
  const brains = (org.staff ?? []).map((s) => String(s.dir ?? s.handle));
7824
7874
  const products = opts.products ? (org.repos ?? []).map((r) => String(r.name)).filter((n) => n !== portal.opsName && !brains.includes(n)) : [];
7825
7875
  const results = await Promise.all([
7826
7876
  ...brains.map(
7827
- (d) => portal.snapshots.refresh(actor.gh, `${portal.org}/${d}`, { force: opts.force }).then((r) => ({ dir: d, ...r }))
7877
+ (d) => portal.snapshots.refresh(actor.gh, `${portal.org}/${d}`, { force: opts.force, report: report3 }).then((r) => ({ dir: d, ...r }))
7828
7878
  ),
7829
7879
  ...products.map(
7830
7880
  (d) => portal.snapshots.refresh(actor.gh, `${portal.org}/${d}`, { force: opts.force, pathsOnly: true }).then((r) => ({ dir: d, ...r }))
@@ -7852,24 +7902,26 @@ var ROUTES = {
7852
7902
  headers: { "content-type": asset.type, "cache-control": "public, max-age=86400" }
7853
7903
  });
7854
7904
  },
7905
+ /* The whole org. With ?progress=1 it is a stream instead: a line per step as it is read from
7906
+ GitHub, then the org, so a first load can say what it is doing rather than "Loading…". */
7855
7907
  "/api/org": async (c) => {
7856
- const { portal, actor } = c;
7857
- const { org, brains } = await ready(c);
7858
- const byRoot = /* @__PURE__ */ new Map();
7859
- await Promise.all(
7860
- brains.map(async (dir) => {
7861
- const repo = `${portal.org}/${dir}`;
7862
- const snap = portal.snapshots.get(repo);
7863
- if (!snap) return;
7864
- byRoot.set(
7865
- join29(portal.ws.root, dir),
7866
- await loadHistory(actor.gh, portal.kv, repo, snap.sha)
7867
- );
7868
- })
7869
- );
7870
- const data = buildExport(portal.ws, org, parseYaml2, { history: loadedHistory(byRoot) });
7871
- const left = unfinished(portal.ws.opsDir, (org.staff ?? []).length);
7872
- return json({ ...data, unfinished: left });
7908
+ if (c.url.searchParams.get("progress") !== "1") return json(await orgExport(c));
7909
+ const { readable, writable } = new TransformStream();
7910
+ const out = writable.getWriter();
7911
+ const line = (o) => out.write(new TextEncoder().encode(`${JSON.stringify(o)}
7912
+ `));
7913
+ const report3 = (repo, step) => {
7914
+ void line({ repo: repo.slice(repo.indexOf("/") + 1), step });
7915
+ };
7916
+ void orgExport(c, report3).then((org) => line({ org })).catch(
7917
+ (err) => line({ error: String(err?.message ?? err), status: err.status })
7918
+ ).finally(() => out.close());
7919
+ return new Response(readable, {
7920
+ headers: {
7921
+ "content-type": "application/x-ndjson; charset=utf-8",
7922
+ "cache-control": "no-store"
7923
+ }
7924
+ });
7873
7925
  },
7874
7926
  /* Every file in the org layer the portal may write: isWritable's allowlist, nothing more. */
7875
7927
  "/api/orglayer": async (c) => {
@@ -8431,6 +8483,24 @@ ${diff}`,
8431
8483
  return new Response(bytes2, { headers: { "content-type": type, "cache-control": "no-store" } });
8432
8484
  }
8433
8485
  };
8486
+ async function orgExport(c, report3) {
8487
+ const { portal, actor } = c;
8488
+ const { org, brains } = await ready(c, { report: report3 });
8489
+ const byRoot = /* @__PURE__ */ new Map();
8490
+ await Promise.all(
8491
+ brains.map(async (dir) => {
8492
+ const repo = `${portal.org}/${dir}`;
8493
+ const snap = portal.snapshots.get(repo);
8494
+ if (!snap) return;
8495
+ report3?.(repo, "reading recent history");
8496
+ byRoot.set(join29(portal.ws.root, dir), await loadHistory(actor.gh, portal.kv, repo, snap.sha));
8497
+ report3?.(repo, "ready");
8498
+ })
8499
+ );
8500
+ const data = buildExport(portal.ws, org, parseYaml2, { history: loadedHistory(byRoot) });
8501
+ const left = unfinished(portal.ws.opsDir, (org.staff ?? []).length);
8502
+ return { ...data, unfinished: left };
8503
+ }
8434
8504
  function knownRepos(portal) {
8435
8505
  const org = readOrg(portal.ws.opsDir, parseYaml2);
8436
8506
  return (org.repos ?? []).map((r) => ({
@@ -8998,7 +9068,17 @@ async function portalCommand(argv) {
8998
9068
  return;
8999
9069
  }
9000
9070
  res.writeHead(answer.status, Object.fromEntries(answer.headers));
9001
- res.end(Buffer.from(await answer.arrayBuffer()));
9071
+ if (!answer.body) {
9072
+ res.end();
9073
+ return;
9074
+ }
9075
+ const reader = answer.body.getReader();
9076
+ for (; ; ) {
9077
+ const { done, value } = await reader.read();
9078
+ if (done) break;
9079
+ res.write(value);
9080
+ }
9081
+ res.end();
9002
9082
  };
9003
9083
  const writeAllowed = (req) => {
9004
9084
  if (req.method !== "POST") return false;
@@ -310,3 +310,16 @@ dialog.sheet.side .sidebar-x .iconbtn{margin-top:-4px}
310
310
  .spinner{width:22px;height:22px;border-radius:50%;border:2px solid var(--line);border-top-color:var(--accent);
311
311
  animation:spin .7s linear infinite}
312
312
  @keyframes spin{to{transform:rotate(360deg)}}
313
+
314
+ /* The first load, while the org is read from GitHub: a row per repository. */
315
+ .loadlist{display:flex;flex-direction:column;gap:2px;max-width:520px}
316
+ .loadrow{display:grid;grid-template-columns:18px 140px 1fr;align-items:center;gap:10px;padding:7px 0;
317
+ font-size:13.5px;border-bottom:1px solid var(--line)}
318
+ .loadrow b{font-weight:500}
319
+ .loadrow .meta{color:var(--ink-dim)}
320
+ .loadmark{width:10px;height:10px;border-radius:50%;border:2px solid var(--accent-dim);
321
+ border-top-color:var(--accent);animation:loadspin .8s linear infinite}
322
+ .loadrow.done .loadmark{animation:none;border-color:var(--accent);background:var(--accent)}
323
+ .loadrow.done .meta{color:var(--ink-faint)}
324
+ @keyframes loadspin{to{transform:rotate(360deg)}}
325
+ @media (prefers-reduced-motion:reduce){.loadmark{animation:none}}
@@ -14,6 +14,36 @@ export const getSession = () =>
14
14
  .catch(() => null);
15
15
 
16
16
  export const getOrg = () => json("/api/org");
17
+
18
+ /**
19
+ * The org, for the first load, saying what it is doing as it goes: `onStep(repo, step)` for each
20
+ * line the server streams, then the org itself. Anything that is not a stream (the setup answer,
21
+ * an older server) is read as plain JSON, so this is never worse than getOrg.
22
+ */
23
+ export async function getOrgWithProgress(onStep) {
24
+ const res = await fetch("/api/org?progress=1", { cache: "no-store" });
25
+ if (res.status === 401) location.reload();
26
+ const type = String(res.headers?.get?.("content-type") ?? "");
27
+ if (!type.includes("ndjson") || !res.body?.pipeThrough) return res.json();
28
+ const reader = res.body.pipeThrough(new TextDecoderStream()).getReader();
29
+ let buf = "";
30
+ for (;;) {
31
+ const { done, value } = await reader.read();
32
+ if (done) break;
33
+ buf += value;
34
+ let nl;
35
+ while ((nl = buf.indexOf("\n")) >= 0) {
36
+ const line = buf.slice(0, nl).trim();
37
+ buf = buf.slice(nl + 1);
38
+ if (!line) continue;
39
+ const msg = JSON.parse(line);
40
+ if (msg.org) return msg.org;
41
+ if (msg.error) throw new Error(msg.error);
42
+ onStep(msg.repo, msg.step);
43
+ }
44
+ }
45
+ throw new Error("The org did not arrive. Reload to try again.");
46
+ }
17
47
  export const getDocs = () => json("/api/docs");
18
48
  export const getInbox = (force) => json("/api/inbox" + (force ? "?refresh=1" : ""));
19
49
  export const getSync = () => json("/api/sync").catch(() => null);
@@ -1,6 +1,6 @@
1
1
  /* The shell: boot, the sidebar, and dispatching a view into <main>. */
2
2
 
3
- import { getOrg, getSession } from "./api.js";
3
+ import { getOrgWithProgress, getSession } from "./api.js";
4
4
  import { $, el, store } from "./dom.js";
5
5
  import { icon, iconHTML, staffIcon } from "./icons.js";
6
6
  import { installLightbox } from "./lightbox.js";
@@ -96,7 +96,7 @@ export async function boot() {
96
96
  /* Before there is a tenant the server answers every data route with `mode: setup`. That is
97
97
  not an error state: it is the first thing a new user ever sees, and the whole page becomes
98
98
  the thing that fixes it. */
99
- const [first, session] = await Promise.all([getOrg(), getSession()]);
99
+ const [first, session] = await Promise.all([loadWithProgress(), getSession()]);
100
100
  S.hosted = session?.hosted ? session : null;
101
101
  if (S.hosted) showSignedIn(S.hosted.login);
102
102
  if (first && first.mode === "setup") {
@@ -409,6 +409,41 @@ function render() {
409
409
  (SCREEN[S.view] ?? viewBrain)(m);
410
410
  }
411
411
 
412
+ /* ------------------------------- first load -------------------------------- */
413
+
414
+ /**
415
+ * The first read of the org, with what it is doing on screen. A cold start reads every
416
+ * repository from GitHub, which can take a while, and "Loading…" for fifteen seconds reads as
417
+ * broken. One row per repository, saying which step it is on, and a tick when it is ready.
418
+ */
419
+ async function loadWithProgress() {
420
+ const main = $("#main");
421
+ const rows = new Map();
422
+ const list = el("div", { className: "loadlist" });
423
+ const head = el("p", { className: "sub", textContent: "Reading the org from GitHub…" });
424
+ // Only once something is actually slow: a warm load is over before this would be read.
425
+ const show = setTimeout(() => main.replaceChildren(head, list), 400);
426
+ try {
427
+ return await getOrgWithProgress((repo, step) => {
428
+ let row = rows.get(repo);
429
+ if (!row) {
430
+ row = el("div", { className: "loadrow" }, [
431
+ el("span", { className: "loadmark" }),
432
+ el("b", { textContent: repo }),
433
+ el("span", { className: "meta" }),
434
+ ]);
435
+ rows.set(repo, row);
436
+ list.append(row);
437
+ }
438
+ const ready = step === "ready" || step === "up to date";
439
+ row.classList.toggle("done", ready);
440
+ row.lastChild.textContent = ready ? "ready" : step;
441
+ });
442
+ } finally {
443
+ clearTimeout(show);
444
+ }
445
+ }
446
+
412
447
  /* -------------------------------- signed in -------------------------------- */
413
448
 
414
449
  /**
@@ -35,11 +35,22 @@ function githubClient(token, opts = {}) {
35
35
  body2 = JSON.stringify(o.body);
36
36
  h["content-type"] = "application/json";
37
37
  }
38
+ const reads = method === "GET" || o.retry === true;
38
39
  let res;
39
- try {
40
- res = await doFetch(url, { method, headers: h, body: body2 });
41
- } catch (err) {
42
- return { ok: false, error: `GitHub could not be reached: ${err.message}` };
40
+ for (let attempt = 0; ; attempt++) {
41
+ try {
42
+ res = await doFetch(url, { method, headers: h, body: body2 });
43
+ } catch (err) {
44
+ if (reads && attempt === 0) continue;
45
+ return { ok: false, error: `GitHub could not be reached: ${err.message}` };
46
+ }
47
+ if (reads && attempt === 0 && [502, 503, 504].includes(res.status)) {
48
+ await res.body?.cancel().catch(() => {
49
+ });
50
+ await new Promise((r) => setTimeout(r, 400));
51
+ continue;
52
+ }
53
+ break;
43
54
  }
44
55
  if (res.status === 304 && kept) {
45
56
  seen.delete(key);
@@ -69,7 +80,8 @@ function githubClient(token, opts = {}) {
69
80
  const res = await request(
70
81
  "POST",
71
82
  "graphql",
72
- { body: { query: query2, variables: vars } }
83
+ // A query reads; a mutation writes, and is never asked twice.
84
+ { body: { query: query2, variables: vars }, retry: !/^\s*mutation\b/.test(query2) }
73
85
  );
74
86
  if (!res.ok) return { ok: false, status: res.status, error: res.error };
75
87
  const errors = res.data?.errors ?? [];
@@ -2206,7 +2218,7 @@ var TIMELINE_PR = `
2206
2218
  ... on ReadyForReviewEvent { actor { login } createdAt }
2207
2219
  ... on ReviewRequestedEvent {
2208
2220
  actor { login } createdAt
2209
- requestedReviewer { ... on User { login } ... on Team { name } }
2221
+ requestedReviewer { __typename ... on User { login } ... on Bot { login } }
2210
2222
  }
2211
2223
  `;
2212
2224
  var CLOSED_FIRST = 30;
@@ -2451,7 +2463,9 @@ function event(e) {
2451
2463
  type: "review-requested",
2452
2464
  actor: who,
2453
2465
  createdAt: at,
2454
- assignee: e.requestedReviewer?.login ?? e.requestedReviewer?.name ?? ""
2466
+ /* A team's name needs the read:org scope, which a portal signed in with `repo` alone does
2467
+ not have, and asking for it fails the whole thread. Its type needs nothing. */
2468
+ assignee: e.requestedReviewer?.login ?? (e.requestedReviewer?.__typename === "Team" ? "a team" : "")
2455
2469
  };
2456
2470
  case "LabeledEvent":
2457
2471
  return { type: "labeled", actor: who, createdAt: at, label: e.label?.name ?? "" };
@@ -2989,6 +3003,8 @@ var Snapshots = class {
2989
3003
  repos = /* @__PURE__ */ new Map();
2990
3004
  inflight = /* @__PURE__ */ new Map();
2991
3005
  checked = /* @__PURE__ */ new Map();
3006
+ /** Who wants to hear how a refresh in flight is going. Every caller that joined it. */
3007
+ listeners = /* @__PURE__ */ new Map();
2992
3008
  get(repo) {
2993
3009
  return this.repos.get(repo);
2994
3010
  }
@@ -3003,11 +3019,16 @@ var Snapshots = class {
3003
3019
  if (!opts.force && this.repos.has(repo) && Date.now() - last < this.freshMs) {
3004
3020
  return Promise.resolve({ ok: true, changed: false });
3005
3021
  }
3022
+ if (opts.report) {
3023
+ if (!this.listeners.has(repo)) this.listeners.set(repo, /* @__PURE__ */ new Set());
3024
+ this.listeners.get(repo).add(opts.report);
3025
+ }
3006
3026
  const running = this.inflight.get(repo);
3007
3027
  if (running) return running;
3008
- const work = this.load(gh, repo, opts.pathsOnly === true).finally(
3009
- () => this.inflight.delete(repo)
3010
- );
3028
+ const work = this.load(gh, repo, opts.pathsOnly === true).finally(() => {
3029
+ this.inflight.delete(repo);
3030
+ this.listeners.delete(repo);
3031
+ });
3011
3032
  this.inflight.set(repo, work);
3012
3033
  return work;
3013
3034
  }
@@ -3028,7 +3049,11 @@ var Snapshots = class {
3028
3049
  if (commit2) snap.sha = commit2;
3029
3050
  void this.kv.put(`blob:${sha}`, text2);
3030
3051
  }
3052
+ say(repo, step) {
3053
+ for (const report of this.listeners.get(repo) ?? []) report(repo, step);
3054
+ }
3031
3055
  async load(gh, repo, pathsOnly) {
3056
+ this.say(repo, "checking for changes");
3032
3057
  const head2 = await gh.request("GET", `repos/${repo}/commits/HEAD`, {
3033
3058
  accept: "application/vnd.github.sha",
3034
3059
  as: "text"
@@ -3040,8 +3065,10 @@ var Snapshots = class {
3040
3065
  const sha = head2.data.trim();
3041
3066
  const before = this.repos.get(repo);
3042
3067
  if (before && before.sha === sha && before.pathsOnly === pathsOnly) {
3068
+ this.say(repo, "up to date");
3043
3069
  return { ok: true, changed: false };
3044
3070
  }
3071
+ this.say(repo, "reading the file list");
3045
3072
  const next = await this.build(gh, repo, sha, pathsOnly, before);
3046
3073
  if (!next.ok) return { ok: false, changed: false, error: next.error, status: next.status };
3047
3074
  this.repos.set(repo, next.snap);
@@ -3090,6 +3117,7 @@ var Snapshots = class {
3090
3117
  })
3091
3118
  );
3092
3119
  if (missing2.length > TARBALL_OVER) {
3120
+ this.say(repo, "downloading the repository");
3093
3121
  const tgz = await gh.request("GET", `repos/${repo}/tarball/${sha}`, {
3094
3122
  as: "bytes"
3095
3123
  });
@@ -3104,6 +3132,9 @@ var Snapshots = class {
3104
3132
  void this.kv.put(`blob:${e.sha}`, text2);
3105
3133
  }
3106
3134
  } else {
3135
+ if (missing2.length) {
3136
+ this.say(repo, `fetching ${missing2.length} changed file${missing2.length === 1 ? "" : "s"}`);
3137
+ }
3107
3138
  await pool2(missing2, 8, async ([, e]) => {
3108
3139
  const res = await gh.request("GET", `repos/${repo}/git/blobs/${e.sha}`, {
3109
3140
  accept: "application/vnd.github.raw",
@@ -3141,6 +3172,7 @@ var Snapshots = class {
3141
3172
  else ask.push(path);
3142
3173
  }
3143
3174
  const [owner, name] = repo.split("/");
3175
+ if (ask.length) this.say(repo, "reading file dates");
3144
3176
  for (let i = 0; i < ask.length; i += DATES_PER_QUERY) {
3145
3177
  const batch = ask.slice(i, i + DATES_PER_QUERY);
3146
3178
  const fields = batch.map(
@@ -3271,19 +3303,24 @@ async function handlePortal(req, portal2, actor) {
3271
3303
  }
3272
3304
  async function ready(c, opts = {}) {
3273
3305
  const { portal: portal2, actor } = c;
3274
- const ops = await portal2.snapshots.refresh(actor.gh, `${portal2.org}/${portal2.opsName}`, opts);
3306
+ const report = opts.report;
3307
+ const ops = await portal2.snapshots.refresh(actor.gh, `${portal2.org}/${portal2.opsName}`, {
3308
+ force: opts.force,
3309
+ report
3310
+ });
3275
3311
  if (!ops.ok) {
3276
3312
  throw new HttpError(
3277
3313
  ops.status === 404 || ops.status === 401 ? ops.status : 502,
3278
3314
  `could not read ${portal2.org}/${portal2.opsName}: ${ops.error}`
3279
3315
  );
3280
3316
  }
3317
+ report?.(`${portal2.org}/${portal2.opsName}`, "ready");
3281
3318
  const org = readOrg(portal2.ws.opsDir, parseYaml2);
3282
3319
  const brains = (org.staff ?? []).map((s) => String(s.dir ?? s.handle));
3283
3320
  const products = opts.products ? (org.repos ?? []).map((r) => String(r.name)).filter((n) => n !== portal2.opsName && !brains.includes(n)) : [];
3284
3321
  const results = await Promise.all([
3285
3322
  ...brains.map(
3286
- (d) => portal2.snapshots.refresh(actor.gh, `${portal2.org}/${d}`, { force: opts.force }).then((r) => ({ dir: d, ...r }))
3323
+ (d) => portal2.snapshots.refresh(actor.gh, `${portal2.org}/${d}`, { force: opts.force, report }).then((r) => ({ dir: d, ...r }))
3287
3324
  ),
3288
3325
  ...products.map(
3289
3326
  (d) => portal2.snapshots.refresh(actor.gh, `${portal2.org}/${d}`, { force: opts.force, pathsOnly: true }).then((r) => ({ dir: d, ...r }))
@@ -3315,24 +3352,26 @@ var ROUTES = {
3315
3352
  headers: { "content-type": asset.type, "cache-control": "public, max-age=86400" }
3316
3353
  });
3317
3354
  },
3355
+ /* The whole org. With ?progress=1 it is a stream instead: a line per step as it is read from
3356
+ GitHub, then the org, so a first load can say what it is doing rather than "Loading…". */
3318
3357
  "/api/org": async (c) => {
3319
- const { portal: portal2, actor } = c;
3320
- const { org, brains } = await ready(c);
3321
- const byRoot = /* @__PURE__ */ new Map();
3322
- await Promise.all(
3323
- brains.map(async (dir) => {
3324
- const repo = `${portal2.org}/${dir}`;
3325
- const snap = portal2.snapshots.get(repo);
3326
- if (!snap) return;
3327
- byRoot.set(
3328
- join17(portal2.ws.root, dir),
3329
- await loadHistory(actor.gh, portal2.kv, repo, snap.sha)
3330
- );
3331
- })
3332
- );
3333
- const data = buildExport(portal2.ws, org, parseYaml2, { history: loadedHistory(byRoot) });
3334
- const left = unfinished(portal2.ws.opsDir, (org.staff ?? []).length);
3335
- return json({ ...data, unfinished: left });
3358
+ if (c.url.searchParams.get("progress") !== "1") return json(await orgExport(c));
3359
+ const { readable, writable } = new TransformStream();
3360
+ const out = writable.getWriter();
3361
+ const line = (o) => out.write(new TextEncoder().encode(`${JSON.stringify(o)}
3362
+ `));
3363
+ const report = (repo, step) => {
3364
+ void line({ repo: repo.slice(repo.indexOf("/") + 1), step });
3365
+ };
3366
+ void orgExport(c, report).then((org) => line({ org })).catch(
3367
+ (err) => line({ error: String(err?.message ?? err), status: err.status })
3368
+ ).finally(() => out.close());
3369
+ return new Response(readable, {
3370
+ headers: {
3371
+ "content-type": "application/x-ndjson; charset=utf-8",
3372
+ "cache-control": "no-store"
3373
+ }
3374
+ });
3336
3375
  },
3337
3376
  /* Every file in the org layer the portal may write: isWritable's allowlist, nothing more. */
3338
3377
  "/api/orglayer": async (c) => {
@@ -3894,6 +3933,24 @@ ${diff}`,
3894
3933
  return new Response(bytes, { headers: { "content-type": type, "cache-control": "no-store" } });
3895
3934
  }
3896
3935
  };
3936
+ async function orgExport(c, report) {
3937
+ const { portal: portal2, actor } = c;
3938
+ const { org, brains } = await ready(c, { report });
3939
+ const byRoot = /* @__PURE__ */ new Map();
3940
+ await Promise.all(
3941
+ brains.map(async (dir) => {
3942
+ const repo = `${portal2.org}/${dir}`;
3943
+ const snap = portal2.snapshots.get(repo);
3944
+ if (!snap) return;
3945
+ report?.(repo, "reading recent history");
3946
+ byRoot.set(join17(portal2.ws.root, dir), await loadHistory(actor.gh, portal2.kv, repo, snap.sha));
3947
+ report?.(repo, "ready");
3948
+ })
3949
+ );
3950
+ const data = buildExport(portal2.ws, org, parseYaml2, { history: loadedHistory(byRoot) });
3951
+ const left = unfinished(portal2.ws.opsDir, (org.staff ?? []).length);
3952
+ return { ...data, unfinished: left };
3953
+ }
3897
3954
  function knownRepos(portal2) {
3898
3955
  const org = readOrg(portal2.ws.opsDir, parseYaml2);
3899
3956
  return (org.repos ?? []).map((r) => ({
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@nanocollective/roster",
3
- "version": "0.1.0-beta.1",
3
+ "version": "0.1.0-beta.3",
4
4
  "description": "An agent-run org, powered by GitHub. Scaffold AI staff members whose brain is a repo.",
5
5
  "type": "module",
6
6
  "bin": {
@@ -310,3 +310,16 @@ dialog.sheet.side .sidebar-x .iconbtn{margin-top:-4px}
310
310
  .spinner{width:22px;height:22px;border-radius:50%;border:2px solid var(--line);border-top-color:var(--accent);
311
311
  animation:spin .7s linear infinite}
312
312
  @keyframes spin{to{transform:rotate(360deg)}}
313
+
314
+ /* The first load, while the org is read from GitHub: a row per repository. */
315
+ .loadlist{display:flex;flex-direction:column;gap:2px;max-width:520px}
316
+ .loadrow{display:grid;grid-template-columns:18px 140px 1fr;align-items:center;gap:10px;padding:7px 0;
317
+ font-size:13.5px;border-bottom:1px solid var(--line)}
318
+ .loadrow b{font-weight:500}
319
+ .loadrow .meta{color:var(--ink-dim)}
320
+ .loadmark{width:10px;height:10px;border-radius:50%;border:2px solid var(--accent-dim);
321
+ border-top-color:var(--accent);animation:loadspin .8s linear infinite}
322
+ .loadrow.done .loadmark{animation:none;border-color:var(--accent);background:var(--accent)}
323
+ .loadrow.done .meta{color:var(--ink-faint)}
324
+ @keyframes loadspin{to{transform:rotate(360deg)}}
325
+ @media (prefers-reduced-motion:reduce){.loadmark{animation:none}}
@@ -14,6 +14,36 @@ export const getSession = () =>
14
14
  .catch(() => null);
15
15
 
16
16
  export const getOrg = () => json("/api/org");
17
+
18
+ /**
19
+ * The org, for the first load, saying what it is doing as it goes: `onStep(repo, step)` for each
20
+ * line the server streams, then the org itself. Anything that is not a stream (the setup answer,
21
+ * an older server) is read as plain JSON, so this is never worse than getOrg.
22
+ */
23
+ export async function getOrgWithProgress(onStep) {
24
+ const res = await fetch("/api/org?progress=1", { cache: "no-store" });
25
+ if (res.status === 401) location.reload();
26
+ const type = String(res.headers?.get?.("content-type") ?? "");
27
+ if (!type.includes("ndjson") || !res.body?.pipeThrough) return res.json();
28
+ const reader = res.body.pipeThrough(new TextDecoderStream()).getReader();
29
+ let buf = "";
30
+ for (;;) {
31
+ const { done, value } = await reader.read();
32
+ if (done) break;
33
+ buf += value;
34
+ let nl;
35
+ while ((nl = buf.indexOf("\n")) >= 0) {
36
+ const line = buf.slice(0, nl).trim();
37
+ buf = buf.slice(nl + 1);
38
+ if (!line) continue;
39
+ const msg = JSON.parse(line);
40
+ if (msg.org) return msg.org;
41
+ if (msg.error) throw new Error(msg.error);
42
+ onStep(msg.repo, msg.step);
43
+ }
44
+ }
45
+ throw new Error("The org did not arrive. Reload to try again.");
46
+ }
17
47
  export const getDocs = () => json("/api/docs");
18
48
  export const getInbox = (force) => json("/api/inbox" + (force ? "?refresh=1" : ""));
19
49
  export const getSync = () => json("/api/sync").catch(() => null);
@@ -1,6 +1,6 @@
1
1
  /* The shell: boot, the sidebar, and dispatching a view into <main>. */
2
2
 
3
- import { getOrg, getSession } from "./api.js";
3
+ import { getOrgWithProgress, getSession } from "./api.js";
4
4
  import { $, el, store } from "./dom.js";
5
5
  import { icon, iconHTML, staffIcon } from "./icons.js";
6
6
  import { installLightbox } from "./lightbox.js";
@@ -96,7 +96,7 @@ export async function boot() {
96
96
  /* Before there is a tenant the server answers every data route with `mode: setup`. That is
97
97
  not an error state: it is the first thing a new user ever sees, and the whole page becomes
98
98
  the thing that fixes it. */
99
- const [first, session] = await Promise.all([getOrg(), getSession()]);
99
+ const [first, session] = await Promise.all([loadWithProgress(), getSession()]);
100
100
  S.hosted = session?.hosted ? session : null;
101
101
  if (S.hosted) showSignedIn(S.hosted.login);
102
102
  if (first && first.mode === "setup") {
@@ -409,6 +409,41 @@ function render() {
409
409
  (SCREEN[S.view] ?? viewBrain)(m);
410
410
  }
411
411
 
412
+ /* ------------------------------- first load -------------------------------- */
413
+
414
+ /**
415
+ * The first read of the org, with what it is doing on screen. A cold start reads every
416
+ * repository from GitHub, which can take a while, and "Loading…" for fifteen seconds reads as
417
+ * broken. One row per repository, saying which step it is on, and a tick when it is ready.
418
+ */
419
+ async function loadWithProgress() {
420
+ const main = $("#main");
421
+ const rows = new Map();
422
+ const list = el("div", { className: "loadlist" });
423
+ const head = el("p", { className: "sub", textContent: "Reading the org from GitHub…" });
424
+ // Only once something is actually slow: a warm load is over before this would be read.
425
+ const show = setTimeout(() => main.replaceChildren(head, list), 400);
426
+ try {
427
+ return await getOrgWithProgress((repo, step) => {
428
+ let row = rows.get(repo);
429
+ if (!row) {
430
+ row = el("div", { className: "loadrow" }, [
431
+ el("span", { className: "loadmark" }),
432
+ el("b", { textContent: repo }),
433
+ el("span", { className: "meta" }),
434
+ ]);
435
+ rows.set(repo, row);
436
+ list.append(row);
437
+ }
438
+ const ready = step === "ready" || step === "up to date";
439
+ row.classList.toggle("done", ready);
440
+ row.lastChild.textContent = ready ? "ready" : step;
441
+ });
442
+ } finally {
443
+ clearTimeout(show);
444
+ }
445
+ }
446
+
412
447
  /* -------------------------------- signed in -------------------------------- */
413
448
 
414
449
  /**