meta-sovereign 0.49.2 → 0.49.4

package/js/src/server/aggregate.js

@@ -27,13 +27,27 @@ const blank = (id) => ({
   open: null,
 });
 
+// A message time as an ISO string. VK and WhatsApp keep unix seconds (a
+// number), which used to break the contact sort and `/api/status` (R-Z8).
+const seenAt = (value) => {
+  if (value === null || value === undefined || value === '') {
+    return null;
+  }
+  const n = Number(value);
+  if (Number.isFinite(n) && !/[-:T]/.test(String(value))) {
+    return new Date(n < 1e12 ? n * 1000 : n).toISOString();
+  }
+  return String(value);
+};
+
 const addMessage = (entry, m) => {
   entry.networks.add(m.source);
   entry.chats.add(m.chat);
   entry.messageCount += 1;
-  const latest = !entry.lastSeen || (m.timestamp ?? '') > entry.lastSeen;
+  const at = seenAt(m.timestamp);
+  const latest = !entry.lastSeen || (at ?? '') > entry.lastSeen;
   if (latest) {
-    entry.lastSeen = m.timestamp ?? null;
+    entry.lastSeen = at;
     entry.name = m.senderName ?? entry.name;
   } else {
     entry.name ??= m.senderName ?? null;

package/js/src/server/github-service.js

@@ -0,0 +1,383 @@
+/**
+ * GitHub as a full inbox, server side (R-Z11).
+ *
+ * Which repositories to pull and how often is a saved setting
+ * (`settings:github`); where each pull stopped is a cursor link
+ * (`cursor:github:<owner/repo>`) written only after the pulled links were
+ * stored (R-Z14), so a failed write repeats the same window. A repository
+ * that fails does not stop the others; a rate limit stops the run and says
+ * when to come back. Notifications are fetched into `notification:github:*`
+ * links and marked read only by an explicit call.
+ *
+ * There is no timer here: `pullJob()` is the job the shared pull runner
+ * (../sources/pull-runner.js, R-Z8) registers as `github`, so "Fetch now" and
+ * the schedule are the runner's. The schedule is off until the user picks
+ * repositories and an interval.
+ */
+
+import { createGithubLive } from '../sources/github.js';
+import { stampSourceLink } from '../sources/index.js';
+import { loadPullState, savePullState } from '../sources/pull-runner.js';
+import { safeMessage } from './redact.js';
+
+const SETTINGS_ID = 'settings:github';
+const cursorId = (key) => `cursor:github:${key}`;
+const REPO = /^[A-Za-z0-9_.-]{1,100}\/[A-Za-z0-9_.-]{1,100}$/;
+const MAX_REPOS = 200;
+const MIN_INTERVAL_MINUTES = 5;
+const MAX_INTERVAL_MINUTES = 24 * 60;
+
+export const DEFAULT_SETTINGS = Object.freeze({
+  repos: [],
+  intervalMinutes: 0,
+  notifications: false,
+  discussions: false,
+});
+
+const secretValue = (link) =>
+  link?.token ?? link?.value ?? link?.body ?? link?.text ?? null;
+
+export const readGithubToken = async (store, { secretId } = {}) => {
+  const ids = [
+    secretId,
+    'secret:github:access-token',
+    'secret:github:token',
+  ].filter(Boolean);
+  for (const id of ids) {
+    const token = secretValue(await store.get(id));
+    if (token) {
+      return token;
+    }
+  }
+  return null;
+};
+
+export const githubLive = async (store, body, ctx) => {
+  const token = body.token ?? (await readGithubToken(store, body));
+  const factory = ctx?.githubLiveFactory ?? createGithubLive;
+  const live = factory({
+    token,
+    owner: body.owner ?? null,
+    repo: body.repo ?? null,
+  });
+  return { token, live };
+};
+
+const invalid = (message) =>
+  Object.assign(new Error(message), { code: 'bad-request' });
+
+/** The settings a request may save, or a `bad-request` error. */
+export const checkSettings = (input = {}) => {
+  const repos = input.repos ?? [];
+  if (
+    !Array.isArray(repos) ||
+    repos.length > MAX_REPOS ||
+    !repos.every((r) => typeof r === 'string' && REPO.test(r))
+  ) {
+    throw invalid('repos must be a list of owner/repo names');
+  }
+  const interval = Number(input.intervalMinutes ?? 0);
+  if (
+    !Number.isInteger(interval) ||
+    (interval !== 0 &&
+      (interval < MIN_INTERVAL_MINUTES || interval > MAX_INTERVAL_MINUTES))
+  ) {
+    throw invalid(
+      `intervalMinutes must be 0 (off) or ${MIN_INTERVAL_MINUTES}..${MAX_INTERVAL_MINUTES}`
+    );
+  }
+  return {
+    repos: [...new Set(repos)],
+    intervalMinutes: interval,
+    notifications: input.notifications === true,
+    discussions: input.discussions === true,
+  };
+};
+
+const errorEntry = (error) => ({
+  error: safeMessage(error),
+  code: error.code ?? 'github-error',
+  ...(error.retryAfter ? { retryAfter: error.retryAfter } : {}),
+});
+
+const putAll = async (store, links) => {
+  for (const link of links) {
+    await store.put(stampSourceLink(link, 'github'));
+  }
+};
+
+// One repository: pull, store, then (only then) move its cursor.
+const fetchRepo = async (env, live, token, key, options) => {
+  const { store, now } = env;
+  const [owner, repo] = key.split('/');
+  const cursor = (await store.get(cursorId(key))) ?? {};
+  const result = await live.pullMessages({
+    owner,
+    repo,
+    token,
+    since: cursor.since ?? null,
+    etags: cursor.etags ?? {},
+  });
+  const found = options.discussions
+    ? await fetchDiscussionsOf(env, live, { owner, repo, token, cursor })
+    : { links: [], since: cursor.discussionsSince ?? null, note: null };
+  await putAll(store, [...result.links, ...found.links]);
+  // After the store write: a failed write above leaves the cursor as it was.
+  await store.put({
+    id: cursorId(key),
+    source: 'github',
+    since: result.since,
+    etags: result.etags,
+    discussionsSince: found.since,
+    fetchedAt: new Date(now()).toISOString(),
+  });
+  return {
+    repo: key,
+    imported: result.links.length + found.links.length,
+    rawCount: result.rawCount,
+    notModified: result.notModified,
+    discussions: options.discussions ? found.links.length : null,
+    ...(found.note ? { discussionsNote: found.note } : {}),
+  };
+};
+
+// Discussions need a token with access; without it the repository still
+// pulls and the reason is reported (a rate limit still stops the run).
+const fetchDiscussionsOf = async (
+  env,
+  live,
+  { owner, repo, token, cursor }
+) => {
+  const startedAt = new Date(env.now()).toISOString();
+  try {
+    const found = await live.fetchDiscussions({
+      owner,
+      repo,
+      token,
+      since: cursor.discussionsSince ?? null,
+    });
+    return { links: found.links, since: startedAt, note: null };
+  } catch (error) {
+    if (error.code === 'rate-limited') {
+      throw error;
+    }
+    return {
+      links: [],
+      since: cursor.discussionsSince ?? null,
+      note: errorEntry(error),
+    };
+  }
+};
+
+// The notification inbox, never faster than the interval GitHub asked for.
+const fetchNotes = async ({ store, now }, live, token) => {
+  const id = cursorId('notifications');
+  const cursor = (await store.get(id)) ?? {};
+  const next = Number(cursor.nextPollAt ?? 0);
+  if (next > now()) {
+    return { skipped: 'poll-interval', nextPollAt: next };
+  }
+  const startedAt = new Date(now()).toISOString();
+  const found = await live.fetchNotifications({
+    token,
+    since: cursor.since ?? null,
+    lastModified: cursor.lastModified ?? null,
+  });
+  await putAll(store, found.links);
+  const pollInterval = found.pollInterval ?? cursor.pollInterval ?? 60;
+  await store.put({
+    id,
+    source: 'github',
+    since: found.notModified ? (cursor.since ?? startedAt) : startedAt,
+    lastModified: found.lastModified,
+    pollInterval,
+    nextPollAt: now() + pollInterval * 1000,
+  });
+  return { imported: found.links.length, notModified: found.notModified };
+};
+
+const storedNotifications = async (store) => {
+  const all = await store.query(
+    (l) => typeof l?.id === 'string' && l.id.startsWith('notification:github:')
+  );
+  return all
+    .filter((l) => !l.deleted)
+    .sort((a, b) => String(b.updatedAt).localeCompare(String(a.updatedAt)));
+};
+
+// A repository that fails does not stop the others; a rate limit does.
+const runAll = async (env, saved, wanted, { token, live }, log) => {
+  const out = { repos: [], notifications: null, rateLimited: null };
+  const limited = (error) => {
+    if (error.code === 'rate-limited') {
+      out.rateLimited = { retryAfter: error.retryAfter };
+    }
+  };
+  for (const key of wanted) {
+    if (out.rateLimited) {
+      out.repos.push({ repo: key, skipped: 'rate-limited' });
+      continue;
+    }
+    try {
+      out.repos.push(await fetchRepo(env, live, token, key, saved));
+    } catch (error) {
+      out.repos.push({ repo: key, ...errorEntry(error) });
+      limited(error);
+      log?.({ level: 'warn', event: 'github_fetch_failed', repo: key });
+    }
+  }
+  if (saved.notifications && !out.rateLimited) {
+    out.notifications = await fetchNotes(env, live, token).catch((error) => {
+      limited(error);
+      return errorEntry(error);
+    });
+  }
+  out.imported = out.repos.reduce((n, r) => n + (r.imported ?? 0), 0);
+  return out;
+};
+
+/**
+ * @param {object} options
+ * @param {object} options.store
+ * @param {object} [options.ctx] server context (`githubLiveFactory`)
+ * @param {() => number} [options.now]
+ * @param {Function} [options.log]
+ */
+export const createGithubService = ({
+  store,
+  ctx = {},
+  now = () => Date.now(),
+  log = null,
+}) => {
+  const env = { store, now };
+  let running = null;
+
+  const settings = async () => {
+    const { repos, intervalMinutes, notifications, discussions } = {
+      ...DEFAULT_SETTINGS,
+      ...((await store.get(SETTINGS_ID)) ?? {}),
+    };
+    return { repos, intervalMinutes, notifications, discussions };
+  };
+
+  const saveSettings = async (input) => {
+    const next = checkSettings(input);
+    await store.put({ id: SETTINGS_ID, source: 'github', ...next });
+    return next;
+  };
+
+  const withLive = async () => {
+    const { token, live } = await githubLive(store, {}, ctx);
+    if (!token) {
+      throw Object.assign(new Error('save a GitHub access token first'), {
+        code: 'no-token',
+      });
+    }
+    return { token, live };
+  };
+
+  const listRepos = async () => {
+    const { token, live } = await withLive();
+    const repos = await live.listRepos({ token });
+    const chosen = new Set((await settings()).repos);
+    return repos.map((r) => ({
+      fullName: r.full_name,
+      private: Boolean(r.private),
+      description: r.description ?? '',
+      updatedAt: r.pushed_at ?? r.updated_at ?? null,
+      selected: chosen.has(r.full_name),
+    }));
+  };
+
+  const run = async (options = {}) => {
+    const saved = await settings();
+    const access = await withLive();
+    const wanted = options.repos?.length ? options.repos : saved.repos;
+    return runAll(env, saved, wanted, access, log);
+  };
+
+  // One run at a time: a second request waits for and shares the first.
+  const fetchNow = (options) => {
+    running ??= run(options).finally(() => {
+      running = null;
+    });
+    return running;
+  };
+
+  const notifications = () => storedNotifications(store);
+
+  // Marking read changes GitHub: only on an explicit call (never by fetching).
+  const markRead = async ({ threadId = null, all = false } = {}) => {
+    const { token, live } = await withLive();
+    if (all) {
+      await live.markAllRead({ token });
+      for (const link of await notifications()) {
+        await store.put({ ...link, unread: false });
+      }
+      return { ok: true, all: true };
+    }
+    await live.markThreadRead(threadId, { token });
+    const link = await store.get(`notification:github:${threadId}`);
+    if (link) {
+      await store.put({ ...link, unread: false });
+    }
+    return { ok: true, threadId: String(threadId) };
+  };
+
+  // The runner's job. A rate limit is remembered (`notBefore` in the pull
+  // state), so neither the schedule nor a click asks GitHub again before it
+  // allows, and the wait is reported instead of looped on.
+  const waitLeft = async () =>
+    Math.ceil(
+      (Number((await loadPullState(store, 'github')).notBefore ?? 0) - now()) /
+        1000
+    );
+  const limitError = (seconds) =>
+    Object.assign(new Error('GitHub rate limit reached; try again later'), {
+      code: 'rate-limited',
+      retryAfter: Math.max(1, seconds),
+    });
+  const pullJob = () => ({
+    ready: async () => Boolean(await readGithubToken(store)),
+    intervalMs: 60_000,
+    firstDelayMs: 30_000,
+    due: async () => {
+      const saved = await settings();
+      if (!saved.intervalMinutes || saved.repos.length === 0) {
+        return false;
+      }
+      const state = await loadPullState(store, 'github');
+      const last = Date.parse(state.lastOkAt ?? '') || 0;
+      return (
+        now() >=
+        Math.max(last + saved.intervalMinutes * 60_000, state.notBefore ?? 0)
+      );
+    },
+    run: async () => {
+      const left = await waitLeft();
+      if (left > 0) {
+        throw limitError(left);
+      }
+      const out = await fetchNow({});
+      if (out.rateLimited) {
+        const seconds = out.rateLimited.retryAfter ?? 60;
+        await savePullState(store, 'github', {
+          notBefore: now() + seconds * 1000,
+        });
+        throw limitError(seconds);
+      }
+      await savePullState(store, 'github', { notBefore: 0 });
+      return { imported: out.imported };
+    },
+  });
+
+  return {
+    settings,
+    saveSettings,
+    listRepos,
+    fetchNow,
+    notifications,
+    markRead,
+    pullJob,
+  };
+};

package/js/src/server/index.js

@@ -104,6 +104,7 @@ import { createGitHubCheck } from '../sources/github-check.js';
 import { handleMetrics } from './metrics.js';
 import { jsonLog } from './log.js';
 import { listenLocal } from './local-tls.js';
+import { handlePullRoutes, pullContext } from './routes-pull.js';
 import { createPeer, attachSyncWebSocket } from '../sync/index.js';
 import { attachSignaling } from '../sync/webrtc-signaling.js';
 import { registerDefaultHandlers } from './handlers-bootstrap.js';
@@ -148,7 +149,9 @@ export const CSP =
   "default-src 'self'; " +
   "script-src 'self' 'wasm-unsafe-eval'; " +
   "style-src 'self'; " +
-  "img-src 'self' data: blob: https://vk.com; " +
+  "img-src 'self' data: blob: https://vk.com " +
+  // VK's picture CDNs, for photos and stickers in VK chats (R-Z8).
+  'https://*.userapi.com https://*.vk-cdn.net https://*.vkuserphoto.ru; ' +
   'frame-src https:; ' +
   "connect-src 'self' https: ws: wss: http://127.0.0.1:* http://localhost:*; " +
   "worker-src 'self'; " +
@@ -576,6 +579,7 @@ const routeOn = async (store, req, res, ctx, url, p) => {
     () => ctx.clientLog?.(req, res, p, ctx) ?? false,
     () => handleAutoImportRoutes(store, req, res, p, ctx),
     () => handleWhatsAppRoutes(store, req, res, p, ctx),
+    () => handlePullRoutes(store, req, res, p, ctx),
     () => handleTelegramRoutes(req, res, p, ctx),
     () => handleMessageRoutes(store, req, res, p, url, ctx),
     () => handleEmailGets(store, req, res, p, url, ctx),
@@ -787,15 +791,14 @@ export const startServer = async ({
   telegramExtraBuild,
   // Tests and demos pass a fake, so GitHub is never asked (R-Y53).
   githubCheck,
-  // The hosted app and whether this server also serves its own copy
-  // (./hosted-ui.js, R-Y58 and R-Y59).
+  // The hosted app, and whether this serves its own copy (./hosted-ui.js).
   uiUrl,
   localUi,
-  // HTTPS on the same port for Safari (./local-tls.js, R-Y64); off
-  // unless asked for, `serve` turns it on.
+  // HTTPS on the same port for Safari (./local-tls.js, R-Y64); `serve` sets it.
   tls,
   // Tests: a fake `(dir) => bytes|null` for the free disk space (R-Z14).
   diskFree,
+  pulls,
 } = {}) => {
   const { store, resolvedArchiveDir, flush, storeHealth } = await initStore({
     providedStore,
@@ -853,13 +856,13 @@ export const startServer = async ({
       later
     ),
     chatIndex,
+    ...pullContext(store, log, pulls, later),
   };
   const server = createHttpServer(store, ctx, log);
   const sockets = trackSockets(server);
-  let bus = null;
-  if (enableHandlers) {
-    bus = registerDefaultHandlers(store, { browser: () => ctx.send });
-  }
+  const bus = enableHandlers
+    ? registerDefaultHandlers(store, { browser: () => ctx.send })
+    : null;
   const { wsHandle, signaling } = attachSockets(server, ctx, store, {
     enableSync,
     node,
@@ -903,6 +906,7 @@ export const startServer = async ({
         ctx.closeAutomations().catch(() => {});
         ctx.autoImport?.close();
         ctx.whatsappSignIn?.stop().catch(() => {});
+        ctx.stopPulls?.();
         ctx.chatIndex.close();
         stopAll(telegramAccount, ctx.telegramAccounts, ctx.telegramExtras);
         wsHandle?.close();

package/js/src/server/routes-github.js

@@ -0,0 +1,98 @@
+/**
+ * The GitHub inbox routes (R-Z11), all under /api/github/:
+ *
+ *   GET  repos                  the user's repositories, `selected` marked
+ *   GET  settings               { repos, intervalMinutes, notifications,
+ *                                 discussions }
+ *   POST settings               the same, saved
+ *   GET  notifications          the stored notification inbox
+ *   POST notifications/read     { threadId } | { all: true } -- the only
+ *                               thing that marks anything read on GitHub
+ *
+ * "Fetch now" and the schedule are the shared pull runner's: POST /api/pull
+ * {source: "github"} (./routes-pull.js). Pull, clone and post-comment stay in
+ * ./routes-mutating.js. Answers are
+ * `{ ok, ... }` or the uniform `{ ok: false, error, code }`.
+ */
+
+import { fail, json, readBody } from './util.js';
+import { isTrustedMutation } from './routes-auto-import.js';
+import { safeMessage } from './redact.js';
+
+const STATUS = {
+  'bad-request': 400,
+  'no-token': 400,
+  'rate-limited': 429,
+  'discussions-unavailable': 403,
+};
+
+const failure = (res, error) => {
+  const code = error.code ?? 'github-error';
+  const status = STATUS[code] ?? (error.status === 401 ? 401 : 502);
+  return (
+    json(res, status, {
+      ok: false,
+      error: safeMessage(error),
+      code,
+      ...(error.retryAfter ? { retryAfter: error.retryAfter } : {}),
+    }) ?? true
+  );
+};
+
+const guarded = async (res, work) => {
+  try {
+    return (await work()) ?? true;
+  } catch (error) {
+    return failure(res, error);
+  }
+};
+
+const mutation = async (req, res, ctx, work) => {
+  if (!isTrustedMutation(req, ctx.allowOrigins)) {
+    return fail(
+      res,
+      403,
+      'forbidden',
+      'GitHub settings must be changed from the meta-sovereign app'
+    );
+  }
+  return guarded(res, async () => work(await readBody(req)));
+};
+
+const READ = {
+  repos: async (service) => ({ repos: await service.listRepos() }),
+  settings: async (service) => ({ settings: await service.settings() }),
+  notifications: async (service) => ({
+    notifications: await service.notifications(),
+  }),
+};
+
+const WRITE = {
+  settings: async (service, body) => ({
+    settings: await service.saveSettings(body),
+  }),
+  'notifications/read': (service, body) =>
+    service.markRead({
+      threadId: body.threadId ?? null,
+      all: body.all === true,
+    }),
+};
+
+export const handleGithubRoutes = async (req, res, p, ctx) => {
+  const prefix = '/api/github/';
+  if (!p.startsWith(prefix) || !ctx.github) {
+    return false;
+  }
+  const name = p.slice(prefix.length);
+  if (req.method === 'GET' && Object.hasOwn(READ, name)) {
+    return guarded(res, async () =>
+      json(res, 200, { ok: true, ...(await READ[name](ctx.github)) })
+    );
+  }
+  if (req.method === 'POST' && Object.hasOwn(WRITE, name)) {
+    return mutation(req, res, ctx, async (body) =>
+      json(res, 200, { ok: true, ...(await WRITE[name](ctx.github, body)) })
+    );
+  }
+  return false;
+};

package/js/src/server/routes-messages.js

@@ -170,6 +170,13 @@ const ACTIONS = {
   },
 };
 
+// An item pulled before its network id was kept is refused (R-Z11).
+const FAILURE_STATUS = {
+  'flood-wait': 429,
+  'unsupported-action': 501,
+  'no-external-id': 409,
+};
+
 const handleAction = async (store, req, res, ctx, name) => {
   const action = ACTIONS[name];
   const body = await readBody(req);
@@ -197,12 +204,7 @@ const handleAction = async (store, req, res, ctx, name) => {
       source: target.source,
       code: error.code ?? null,
     });
-    const status =
-      error.code === 'flood-wait'
-        ? 429
-        : error.code === 'unsupported-action'
-          ? 501
-          : 502;
+    const status = FAILURE_STATUS[error.code] ?? 502;
     return (
       json(res, status, {
         error: error.message,

package/js/src/server/routes-mutating.js

@@ -17,7 +17,8 @@ import {
   stampSourceLink,
 } from '../sources/index.js';
 import { emailFailure, handleEmailPull, sendEmail } from './routes-email.js';
-import { createGithubLive } from '../sources/github.js';
+import { githubLive } from './github-service.js';
+import { handleGithubRoutes } from './routes-github.js';
 import { createUpworkLive } from '../sources/upwork.js';
 import { createPeoplePerHourLive } from '../sources/peopleperhour.js';
 import {
@@ -523,33 +524,6 @@ const handleEmail = async (store, req, res, p, ctx) => {
   return false;
 };
 
-const readGithubToken = async (store, { secretId } = {}) => {
-  const ids = [
-    secretId,
-    'secret:github:access-token',
-    'secret:github:token',
-  ].filter(Boolean);
-  for (const id of ids) {
-    const link = await store.get(id);
-    const token = link?.token ?? link?.value ?? link?.body ?? link?.text;
-    if (token) {
-      return token;
-    }
-  }
-  return null;
-};
-
-const githubLive = async (store, body, ctx) => {
-  const token = body.token ?? (await readGithubToken(store, body));
-  const factory = ctx?.githubLiveFactory ?? createGithubLive;
-  const live = factory({
-    token,
-    owner: body.owner ?? null,
-    repo: body.repo ?? null,
-  });
-  return { token, live };
-};
-
 const pullGithub = async (store, body, ctx) => {
   const { token, live } = await githubLive(store, body, ctx);
   const result = await live.pullMessages({ ...body, token });
@@ -584,10 +558,18 @@ const cloneGithub = async (store, body, ctx) => {
 
 const postGithubComment = async (store, body, ctx) => {
   const { token, live } = await githubLive(store, body, ctx);
-  const result = await live.post(body.content ?? body.message ?? body, {
-    ...body,
-    token,
-  });
+  let content = body.content ?? body.message ?? body;
+  // A reply names the message it answers; its saved GitHub id picks the
+  // endpoint (a review comment has its own reply route) (R-Z11).
+  if (typeof body.replyTo === 'string' && body.replyTo.startsWith('msg:')) {
+    const replyToLink = await store.get(body.replyTo);
+    content = {
+      ...(typeof content === 'string' ? { text: content } : content),
+      chat: body.chat ?? replyToLink?.chat,
+      replyToLink,
+    };
+  }
+  const result = await live.post(content, { ...body, token });
   return { source: 'github', result };
 };
 
@@ -805,6 +787,7 @@ export const handleMutatingRoutes = async (store, req, res, p, url, ctx) =>
   (await handleOutreach(store, req, res, p, ctx)) ||
   (await handleEmail(store, req, res, p, ctx)) ||
   (await handleGithub(store, req, res, p, ctx)) ||
+  (await handleGithubRoutes(req, res, p, ctx)) ||
   (await handleUpwork(store, req, res, p, ctx)) ||
   (await handlePeoplePerHour(store, req, res, p, ctx)) ||
   (await handleHardening(store, req, res, p, ctx));