meta-sovereign 0.49.2 → 0.49.4

package/js/src/server/routes-pull.js

@@ -0,0 +1,163 @@
+/**
+ * Inbox pulls for the networks that have no live connection of their own
+ * (R-Z8): the Connections card's "Fetch now" button and the background
+ * schedule behind it.
+ *
+ * Its routes:
+ *   POST /api/pull {source}   -> { ok, source, imported, lastOkAt }
+ *                                 or { ok: false, error, code } with `code` one of
+ *                                 unsupported | not-connected | rate-limited
+ *                                 (HTTP 429, at most one fetch per 15 s per
+ *                                 network, `retryAfterMs`) | the network's own code
+ *   GET  /api/pull/status     -> { ok, sources: { <source>: { connected,
+ *                                 lastOkAt, lastError } } }
+ *
+ * The same `createPullRunner` (../sources/pull-runner.js) serves every
+ * network: add a job to `pullJobs` to give another network a button and a
+ * schedule. VK also keeps a Long Poll running (../sources/vk-sync.js).
+ */
+
+import { createPullRunner, loadPullState } from '../sources/pull-runner.js';
+import { createVkInbox } from '../sources/vk-sync.js';
+import { liveOptionsFrom, pullLiveInto } from '../sources/index.js';
+import { createUpworkLive } from '../sources/upwork.js';
+import { createPeoplePerHourLive } from '../sources/peopleperhour.js';
+import { createGithubService } from './github-service.js';
+import { fail, json, readBody } from './util.js';
+
+const MINUTE = 60_000;
+
+const connected = (store, source) => async () =>
+  Boolean((await liveOptionsFrom(store, source)).token);
+
+// `live` for a network whose test double or host is given; the saved token
+// still comes from the store.
+const liveOf = (make, given) => (given ? make(given) : undefined);
+
+/** The jobs: network -> { ready, run, intervalMs }. */
+export const pullJobs = (store, { vkInbox, github, options = {} }) => ({
+  ...(github ? { github: github.pullJob() } : {}),
+  vk: {
+    ready: vkInbox.ready,
+    run: vkInbox.fetch,
+    intervalMs: 10 * MINUTE,
+    firstDelayMs: 30_000,
+  },
+  upwork: {
+    ready: connected(store, 'upwork'),
+    run: () =>
+      pullLiveInto(store, 'upwork', {
+        live: liveOf(createUpworkLive, options.upwork),
+      }),
+    intervalMs: 5 * MINUTE,
+    firstDelayMs: 30_000,
+  },
+  peopleperhour: {
+    ready: connected(store, 'peopleperhour'),
+    run: () =>
+      pullLiveInto(store, 'peopleperhour', {
+        live: liveOf(createPeoplePerHourLive, options.peopleperhour),
+      }),
+    intervalMs: 5 * MINUTE,
+    firstDelayMs: 30_000,
+  },
+});
+
+/**
+ * The pull services for a server: `{ pullRunner, vkInbox, startAll, stopAll }`.
+ * `options` (tests): `{ vk: {baseUrl, fetchImpl, wait}, upwork: {baseUrl},
+ * peopleperhour: {baseUrl}, minGapMs, intervals }`; `false` turns it all off.
+ */
+export const createPullServices = (store, log, options = {}, github = null) => {
+  const vkInbox = createVkInbox({ store, log, ...options.vk });
+  const jobs = pullJobs(store, { vkInbox, github, options });
+  for (const [source, intervalMs] of Object.entries(options.intervals ?? {})) {
+    jobs[source].intervalMs = intervalMs;
+    jobs[source].firstDelayMs = intervalMs;
+  }
+  const pullRunner = createPullRunner({
+    store,
+    jobs,
+    log,
+    minGapMs: options.minGapMs,
+  });
+  return {
+    pullRunner,
+    vkInbox,
+    start: () => {
+      pullRunner.start();
+      vkInbox.start();
+    },
+    stop: () => {
+      pullRunner.stop();
+      vkInbox.stop();
+    },
+  };
+};
+
+const statusOf = async (store, ctx) => {
+  const sources = {};
+  for (const source of ctx.pullRunner.sources()) {
+    const state = await loadPullState(store, source);
+    sources[source] = {
+      connected: await ctx.pullRunner.connected(source),
+      lastOkAt: state.lastOkAt ?? null,
+      lastError: state.lastError ?? null,
+    };
+  }
+  return { ok: true, sources };
+};
+
+const sourceOf = (body) =>
+  typeof body.source === 'string' && /^[a-z]{2,30}$/.test(body.source)
+    ? body.source
+    : null;
+
+export const handlePullRoutes = async (store, req, res, p, ctx) => {
+  if (!ctx.pullRunner) {
+    return false;
+  }
+  if (p === '/api/pull/status' && req.method === 'GET') {
+    return json(res, 200, await statusOf(store, ctx)) ?? true;
+  }
+  if (p !== '/api/pull' || req.method !== 'POST') {
+    return false;
+  }
+  const source = sourceOf(await readBody(req));
+  if (!source) {
+    return fail(res, 400, 'bad-request', 'a "source" is needed') ?? true;
+  }
+  const outcome = await ctx.pullRunner.fetchNow(source);
+  if (outcome.ok) {
+    return json(res, 200, outcome) ?? true;
+  }
+  const status =
+    { 'rate-limited': 429, unsupported: 404, 'not-connected': 409 }[
+      outcome.code
+    ] ?? 502;
+  return json(res, status, outcome) ?? true;
+};
+
+/** What a server keeps for the pulls; they start with the other `later` services. */
+export const pullContext = (store, log, options, later) => {
+  // GitHub's picker routes work even when the pulls are off; its schedule is
+  // one more job of the shared runner (R-Z11).
+  // Tests give `pulls.githubLiveFactory`, a fake createGithubLive.
+  const githubLiveFactory = options?.githubLiveFactory ?? null;
+  const github = createGithubService({
+    store,
+    ctx: { githubLiveFactory },
+    log,
+  });
+  const given = { githubLiveFactory, github };
+  if (options === false) {
+    return { ...given, pullRunner: null };
+  }
+  const services = createPullServices(store, log, options ?? {}, github);
+  later.push(services.start);
+  return {
+    ...given,
+    pullRunner: services.pullRunner,
+    stopPulls: services.stop,
+  };
+};

package/js/src/sources/auto-import/files.js

@@ -25,7 +25,11 @@ const toArchive = (recipe, name, text, context = {}) => {
 };
 
 const parseEntry = async (provider, recipe, name, data, context) => {
-  const archive = toArchive(recipe, name, data.toString('utf8'), context);
+  // A recipe may decode its own bytes (VK's Windows-1251 pages, R-Z8).
+  const text = recipe.decode
+    ? recipe.decode(name, data)
+    : data.toString('utf8');
+  const archive = toArchive(recipe, name, text, context);
   return (await getSource(provider).parseArchive(archive)) ?? [];
 };
 

package/js/src/sources/auto-import/recipes.js

@@ -23,6 +23,11 @@
 
 import { parseCsv } from './archive.js';
 import { chatNameFromFile } from '../whatsapp-ids.js';
+import {
+  decodeVkHtml,
+  parseVkHtmlPage,
+  peerFromArchivePath,
+} from '../vk-html-archive.js';
 
 // X archives ship data as `window.YTD.<name>.part0 = [ ... ]`.
 const ytdPayload = (text) => JSON.parse(text.replace(/^[^=[]*=\s*/, ''));
@@ -64,6 +69,14 @@ const linkedinToArchive = (_name, text) => ({
   ),
 });
 
+const vkDecode = (name, bytes) =>
+  /\.html$/i.test(name) ? decodeVkHtml(bytes) : bytes.toString('utf8');
+
+const vkToArchive = (name, text) =>
+  /\.html$/i.test(name)
+    ? parseVkHtmlPage(text, { peer: peerFromArchivePath(name) })
+    : JSON.parse(text);
+
 export const autoImportRecipes = {
   email: {
     url: 'https://takeout.google.com/settings/takeout/custom/gmail',
@@ -87,7 +100,11 @@ export const autoImportRecipes = {
   vk: {
     url: 'https://vk.com/data_protection?section=rules',
     domains: ['vk.com', 'vk.ru'],
-    files: /messages.*\.json$/i,
+    // The JSON of konard/vk-export, or the pages of VK's own archive:
+    // messages/<peer>/messages<N>.html in Windows-1251 (R-Z8).
+    files: /(messages.*\.json|messages\/[^/]+\/messages\d*\.html)$/i,
+    decode: vkDecode,
+    toArchive: vkToArchive,
   },
   x: {
     url: 'https://x.com/settings/download_your_data',

package/js/src/sources/github-actions.js

@@ -0,0 +1,162 @@
+/**
+ * What the user does to a GitHub item: reply, edit, delete, react (R-Z11).
+ *
+ * Every action names the object by the reference kept at pull time
+ * (`link.externalId`, ./github-ref.js) -- never by parsing the link id --
+ * so an issue comment, a review comment and an issue body each reach their
+ * own endpoint. An item pulled before the reference existed is refused with
+ * `no-external-id`; nothing is sent and the browser is not tried.
+ */
+
+import { codedError, requestJson } from './http.js';
+import { requireRef } from './github-ref.js';
+import { replyToDiscussion } from './github-inbox.js';
+
+export const GITHUB_EMOJI_TO_REACTION = {
+  '👍': '+1',
+  '👎': '-1',
+  '😄': 'laugh',
+  '🎉': 'hooray',
+  '😕': 'confused',
+  '❤️': 'heart',
+  '🚀': 'rocket',
+  '👀': 'eyes',
+};
+
+const api = (ctx, path) => `${ctx.baseUrl}${path}`;
+const repoPath = (ref) => `/repos/${ref.owner}/${ref.repo}`;
+// Issue and pull request conversation comments live under /issues, the
+// inline comments on the diff under /pulls.
+const commentPath = (ref) =>
+  `${repoPath(ref)}/${ref.kind === 'review-comment' ? 'pulls' : 'issues'}/comments/${ref.id}`;
+
+const send = (ctx, method, path, options, body) =>
+  requestJson(ctx.fetchImpl, api(ctx, path), {
+    method,
+    headers: ctx.headers(options.token),
+    ...(body === undefined ? {} : { body }),
+  });
+
+// A bare numeric id with an owner and repo is the old programmatic call.
+const legacyRef = (ctx, id, options) => {
+  const { owner, repo } = ctx.requireRepo(options);
+  return { kind: 'issue-comment', id, owner, repo };
+};
+
+const refFor = (ctx, { id, link }, options, action) =>
+  link ? requireRef(link, action) : legacyRef(ctx, id, options);
+
+// An issue or pull request is shown as "title, blank line, body".
+const titleAndBody = (text) => {
+  const [title, ...rest] = String(text).split(/\n\s*\n/);
+  return { title: title.trim(), body: rest.join('\n\n') };
+};
+
+export const editGithubComment = async (ctx, params = {}, options = {}) => {
+  const ref = refFor(ctx, params, options, 'edit');
+  if (ref.kind === 'issue' || ref.kind === 'pr') {
+    return send(
+      ctx,
+      'PATCH',
+      `${repoPath(ref)}/issues/${ref.number}`,
+      options,
+      titleAndBody(params.text)
+    );
+  }
+  return send(ctx, 'PATCH', commentPath(ref), options, { body: params.text });
+};
+
+export const deleteGithubComments = async (ctx, params = {}, options = {}) => {
+  const refs = params.links
+    ? params.links.map((link) => requireRef(link, 'delete'))
+    : (params.ids ?? []).map((id) => legacyRef(ctx, id, options));
+  let deleted = 0;
+  for (const ref of refs) {
+    await send(ctx, 'DELETE', commentPath(ref), options);
+    deleted += 1;
+  }
+  return { ok: true, deleted };
+};
+
+export const reactGithubComment = async (ctx, params = {}, options = {}) => {
+  const content = GITHUB_EMOJI_TO_REACTION[params.emoji];
+  if (!content) {
+    throw Object.assign(
+      codedError(
+        params.emoji
+          ? `github: unsupported reaction emoji: ${params.emoji}`
+          : 'github: taking a reaction back needs its id and is not supported',
+        'unsupported-action'
+      ),
+      { fallback: false }
+    );
+  }
+  const ref = refFor(ctx, params, options, 'react');
+  return send(ctx, 'POST', `${commentPath(ref)}/reactions`, options, {
+    content,
+  });
+};
+
+// `owner/repo#12` -- a chat of an issue or pull request.
+const CHAT = /^([^/#\s]+)\/([^#\s]+)#(\d+)$/;
+const DISCUSSION_CHAT = /^([^/#\s]+)\/([^#\s]+)#discussion:(\d+)$/;
+
+const textOf = (content) =>
+  typeof content === 'string'
+    ? content
+    : (content?.text ?? content?.body ?? '');
+
+const chatOf = (content) =>
+  content && typeof content === 'object' ? String(content.chat ?? '') : '';
+
+const issueNumberOf = (content, options, parsed, reply) => {
+  const number =
+    options.issueNumber ?? content?.issueNumber ?? parsed?.[3] ?? reply?.number;
+  if (!number) {
+    throw codedError(
+      'github post requires an issue or pull request number (options.issueNumber, or a chat like owner/repo#12)',
+      'unsupported-chat'
+    );
+  }
+  return number;
+};
+
+// The issue or pull request a post goes to: options, the reply's saved
+// reference, or the chat `owner/repo#12`, in that order.
+const issueEndpoint = (ctx, content, options) => {
+  const parsed = chatOf(content).match(CHAT);
+  const reply = content?.replyToLink?.externalId;
+  const { owner, repo } = ctx.requireRepo({
+    owner: options.owner ?? parsed?.[1] ?? reply?.owner,
+    repo: options.repo ?? parsed?.[2] ?? reply?.repo,
+  });
+  const number = issueNumberOf(content, options, parsed, reply);
+  return `/repos/${owner}/${repo}/issues/${number}/comments`;
+};
+
+// Where a post goes: a review comment's reply endpoint, or the issue.
+const targetOf = (ctx, content, options) => {
+  if (content?.replyToLink?.externalId?.kind === 'review-comment') {
+    const ref = requireRef(content.replyToLink, 'reply');
+    return `${repoPath(ref)}/pulls/${ref.number}/comments/${ref.id}/replies`;
+  }
+  return issueEndpoint(ctx, content, options);
+};
+
+const isDiscussionPost = (content) =>
+  DISCUSSION_CHAT.test(chatOf(content)) ||
+  Boolean(content?.replyToLink?.externalId?.kind?.startsWith('discussion'));
+
+export const postGithubComment = async (ctx, content, options = {}) => {
+  const text = textOf(content);
+  if (!text) {
+    throw new Error('github post text is required');
+  }
+  if (isDiscussionPost(content)) {
+    const discussion = chatOf(content).match(DISCUSSION_CHAT);
+    return replyToDiscussion(ctx, { content, text, discussion }, options);
+  }
+  return send(ctx, 'POST', targetOf(ctx, content, options), options, {
+    body: text,
+  });
+};

package/js/src/sources/github-http.js

@@ -0,0 +1,266 @@
+/**
+ * Reading from the GitHub API politely (R-Z11).
+ *
+ *   - Every list follows the `Link: rel="next"` header until there is no
+ *     next page; a visited-url guard and a very high safety cap stop a
+ *     server that loops (no page counting).
+ *   - `ETag` / `Last-Modified` come back so the caller can ask again with
+ *     `If-None-Match` / `If-Modified-Since`; a `304` costs no rate limit.
+ *   - `x-ratelimit-remaining` / `x-ratelimit-reset` and the secondary limit's
+ *     `retry-after` are respected: a short wait is slept (and the request
+ *     repeated, a read is safe to repeat), a long one is reported as a
+ *     `rate-limited` error carrying `retryAfter` seconds. Never a crash and
+ *     never a busy loop.
+ *
+ * Only reads go through here; writes keep `requestJson` (./http.js), which
+ * marks an ambiguous outcome and is never repeated.
+ */
+
+import { codedError, timedRequest } from './http.js';
+
+const CODES = { 401: 'unauthorized', 404: 'not-found' };
+const SAFETY_MAX_PAGES = 2000;
+const DEFAULT_MAX_WAIT_MS = 60_000;
+const DEFAULT_SECONDARY_WAIT_MS = 60_000;
+
+export const parseLinkHeader = (header) => {
+  const out = {};
+  if (!header) {
+    return out;
+  }
+  for (const part of String(header).split(',')) {
+    const match = part.match(/<([^>]+)>;\s*rel="?([^";\s]+)"?/);
+    if (match) {
+      out[match[2]] = match[1];
+    }
+  }
+  return out;
+};
+
+export const headerValue = (headers, name) => {
+  if (!headers) {
+    return null;
+  }
+  if (typeof headers.get === 'function') {
+    return headers.get(name) ?? headers.get(name.toLowerCase()) ?? null;
+  }
+  return headers[name] ?? headers[name.toLowerCase()] ?? null;
+};
+
+const numberHeader = (headers, name) => {
+  const raw = headerValue(headers, name);
+  const value = raw === null ? NaN : Number(raw);
+  return Number.isFinite(value) ? value : null;
+};
+
+const parseBody = (text) => {
+  if (!text) {
+    return null;
+  }
+  try {
+    return JSON.parse(text);
+  } catch {
+    return text;
+  }
+};
+
+const itemsOf = (body) => {
+  if (Array.isArray(body)) {
+    return body;
+  }
+  return body ? [body] : [];
+};
+
+// Add a page's items; true when the sorted list reached the cursor.
+const takeItems = (page, items, until) => {
+  for (const item of itemsOf(page.body)) {
+    if (until?.(item)) {
+      return true;
+    }
+    items.push(item);
+  }
+  return false;
+};
+
+const isRateLimited = (res, text) => {
+  if (res.status === 429) {
+    return true;
+  }
+  if (res.status !== 403) {
+    return false;
+  }
+  return (
+    headerValue(res.headers, 'retry-after') !== null ||
+    headerValue(res.headers, 'x-ratelimit-remaining') === '0' ||
+    /rate limit|abuse/i.test(text)
+  );
+};
+
+const rateLimitedError = (seconds, scope) =>
+  Object.assign(
+    codedError(
+      `GitHub rate limit reached (${scope}); try again in ${seconds} s`,
+      'rate-limited'
+    ),
+    { retryAfter: seconds, status: 429 }
+  );
+
+/**
+ * @param {object} options
+ * @param {Function} options.fetchImpl
+ * @param {() => number} [options.now] milliseconds
+ * @param {(ms: number) => Promise<void>} [options.sleep]
+ * @param {number} [options.maxWaitMs] the longest wait slept; a longer one
+ *   is reported to the caller instead
+ * @param {number} [options.retries] repeats after a rate-limit wait
+ */
+export const createGithubHttp = ({
+  fetchImpl,
+  now = () => Date.now(),
+  sleep = (ms) => new Promise((resolve) => globalThis.setTimeout(resolve, ms)),
+  maxWaitMs = DEFAULT_MAX_WAIT_MS,
+  retries = 2,
+  timeoutMs,
+} = {}) => {
+  const limit = { remaining: null, resetAt: 0 };
+
+  const learn = (res) => {
+    const remaining = numberHeader(res.headers, 'x-ratelimit-remaining');
+    const reset = numberHeader(res.headers, 'x-ratelimit-reset');
+    if (remaining !== null) {
+      limit.remaining = remaining;
+    }
+    if (reset !== null) {
+      limit.resetAt = reset * 1000;
+    }
+  };
+
+  // How long to stay quiet after a refusal.
+  const waitAfter = (res) => {
+    const retry = numberHeader(res.headers, 'retry-after');
+    if (retry !== null) {
+      return { ms: retry * 1000, scope: 'secondary' };
+    }
+    if (limit.remaining === 0 && limit.resetAt > now()) {
+      return { ms: limit.resetAt - now(), scope: 'primary' };
+    }
+    return { ms: DEFAULT_SECONDARY_WAIT_MS, scope: 'secondary' };
+  };
+
+  const pauseOrThrow = async (ms, scope) => {
+    if (ms > maxWaitMs) {
+      throw rateLimitedError(Math.ceil(ms / 1000), scope);
+    }
+    await sleep(Math.max(ms, 0));
+  };
+
+  // Spent budget: wait for the reset (or say how long) before asking.
+  const beforeRequest = async () => {
+    if (limit.remaining === 0 && limit.resetAt > now()) {
+      await pauseOrThrow(limit.resetAt - now() + 1000, 'primary');
+      limit.remaining = null;
+    }
+  };
+
+  const once = (url, { headers, method, body, etag, lastModified }) => {
+    const conditional = {
+      ...headers,
+      ...(etag ? { 'If-None-Match': etag } : {}),
+      ...(lastModified ? { 'If-Modified-Since': lastModified } : {}),
+    };
+    return timedRequest(
+      fetchImpl,
+      url,
+      {
+        method,
+        headers: conditional,
+        ...(body !== undefined ? { body: JSON.stringify(body) } : {}),
+      },
+      { timeoutMs, write: false },
+      async (res) => ({ res, text: res.status === 304 ? '' : await res.text() })
+    );
+  };
+
+  // What a good answer carries: the body and the headers worth keeping.
+  const answer = (res, text, settings) => ({
+    status: res.status,
+    notModified: res.status === 304,
+    body: res.status === 304 ? null : parseBody(text),
+    etag: headerValue(res.headers, 'ETag') ?? settings.etag ?? null,
+    lastModified:
+      headerValue(res.headers, 'Last-Modified') ??
+      (res.status === 304 ? (settings.lastModified ?? null) : null),
+    next:
+      res.status === 304
+        ? null
+        : (parseLinkHeader(headerValue(res.headers, 'Link')).next ?? null),
+    pollInterval: numberHeader(res.headers, 'X-Poll-Interval'),
+  });
+
+  /**
+   * One read. Resolves `{ status, body, notModified, etag, lastModified,
+   * next, pollInterval }`; throws on a refusal (`rate-limited` carries
+   * `retryAfter`).
+   */
+  const request = async (url, options = {}) => {
+    const settings = { method: 'GET', ...options };
+    for (let attempt = 0; ; attempt += 1) {
+      await beforeRequest();
+      const { res, text } = await once(url, settings);
+      learn(res);
+      if (res.status === 304 || res.ok) {
+        return answer(res, text, settings);
+      }
+      if (isRateLimited(res, text)) {
+        const { ms, scope } = waitAfter(res);
+        if (attempt >= retries || ms > maxWaitMs) {
+          throw rateLimitedError(Math.ceil(ms / 1000), scope);
+        }
+        await sleep(ms);
+        continue;
+      }
+      throw Object.assign(
+        new Error(`HTTP ${res.status} ${res.statusText}: ${text}`),
+        { status: res.status, code: CODES[res.status] ?? 'github-error' }
+      );
+    }
+  };
+
+  /**
+   * Every page of a list. `until(item)` true on a sorted list ends the walk
+   * (the rest is older than the cursor). The first page may be conditional
+   * (`etag`): then `notModified` is true and `items` is empty.
+   */
+  const pages = async (url, options = {}) => {
+    const { until, maxPages = SAFETY_MAX_PAGES, ...first } = options;
+    const items = [];
+    const seen = new Set();
+    let target = url;
+    let head = null;
+    let count = 0;
+    while (target && !seen.has(target) && count < maxPages) {
+      seen.add(target);
+      const page = await request(
+        target,
+        count === 0
+          ? first
+          : { headers: first.headers, method: first.method ?? 'GET' }
+      );
+      head ??= page;
+      count += 1;
+      if (page.notModified) {
+        return { items, notModified: true, etag: head.etag, head };
+      }
+      target = takeItems(page, items, until) ? null : page.next;
+    }
+    return {
+      items,
+      notModified: false,
+      etag: head?.etag ?? null,
+      lastModified: head?.lastModified ?? null,
+      head,
+    };
+  };
+
+  return { request, pages, limit };
+};