meta-sovereign 0.49.2 → 0.49.3

package/js/src/server/aggregate.js

@@ -27,13 +27,27 @@ const blank = (id) => ({
   open: null,
 });
 
+// A message time as an ISO string. VK and WhatsApp keep unix seconds (a
+// number), which used to break the contact sort and `/api/status` (R-Z8).
+const seenAt = (value) => {
+  if (value === null || value === undefined || value === '') {
+    return null;
+  }
+  const n = Number(value);
+  if (Number.isFinite(n) && !/[-:T]/.test(String(value))) {
+    return new Date(n < 1e12 ? n * 1000 : n).toISOString();
+  }
+  return String(value);
+};
+
 const addMessage = (entry, m) => {
   entry.networks.add(m.source);
   entry.chats.add(m.chat);
   entry.messageCount += 1;
-  const latest = !entry.lastSeen || (m.timestamp ?? '') > entry.lastSeen;
+  const at = seenAt(m.timestamp);
+  const latest = !entry.lastSeen || (at ?? '') > entry.lastSeen;
   if (latest) {
-    entry.lastSeen = m.timestamp ?? null;
+    entry.lastSeen = at;
     entry.name = m.senderName ?? entry.name;
   } else {
     entry.name ??= m.senderName ?? null;

package/js/src/server/index.js

@@ -104,6 +104,7 @@ import { createGitHubCheck } from '../sources/github-check.js';
 import { handleMetrics } from './metrics.js';
 import { jsonLog } from './log.js';
 import { listenLocal } from './local-tls.js';
+import { handlePullRoutes, pullContext } from './routes-pull.js';
 import { createPeer, attachSyncWebSocket } from '../sync/index.js';
 import { attachSignaling } from '../sync/webrtc-signaling.js';
 import { registerDefaultHandlers } from './handlers-bootstrap.js';
@@ -148,7 +149,9 @@ export const CSP =
   "default-src 'self'; " +
   "script-src 'self' 'wasm-unsafe-eval'; " +
   "style-src 'self'; " +
-  "img-src 'self' data: blob: https://vk.com; " +
+  "img-src 'self' data: blob: https://vk.com " +
+  // VK's picture CDNs, for photos and stickers in VK chats (R-Z8).
+  'https://*.userapi.com https://*.vk-cdn.net https://*.vkuserphoto.ru; ' +
   'frame-src https:; ' +
   "connect-src 'self' https: ws: wss: http://127.0.0.1:* http://localhost:*; " +
   "worker-src 'self'; " +
@@ -576,6 +579,7 @@ const routeOn = async (store, req, res, ctx, url, p) => {
     () => ctx.clientLog?.(req, res, p, ctx) ?? false,
     () => handleAutoImportRoutes(store, req, res, p, ctx),
     () => handleWhatsAppRoutes(store, req, res, p, ctx),
+    () => handlePullRoutes(store, req, res, p, ctx),
     () => handleTelegramRoutes(req, res, p, ctx),
     () => handleMessageRoutes(store, req, res, p, url, ctx),
     () => handleEmailGets(store, req, res, p, url, ctx),
@@ -787,15 +791,14 @@ export const startServer = async ({
   telegramExtraBuild,
   // Tests and demos pass a fake, so GitHub is never asked (R-Y53).
   githubCheck,
-  // The hosted app and whether this server also serves its own copy
-  // (./hosted-ui.js, R-Y58 and R-Y59).
+  // The hosted app, and whether this serves its own copy (./hosted-ui.js).
   uiUrl,
   localUi,
-  // HTTPS on the same port for Safari (./local-tls.js, R-Y64); off
-  // unless asked for, `serve` turns it on.
+  // HTTPS on the same port for Safari (./local-tls.js, R-Y64); `serve` sets it.
   tls,
   // Tests: a fake `(dir) => bytes|null` for the free disk space (R-Z14).
   diskFree,
+  pulls,
 } = {}) => {
   const { store, resolvedArchiveDir, flush, storeHealth } = await initStore({
     providedStore,
@@ -853,13 +856,13 @@ export const startServer = async ({
       later
     ),
     chatIndex,
+    ...pullContext(store, log, pulls, later),
   };
   const server = createHttpServer(store, ctx, log);
   const sockets = trackSockets(server);
-  let bus = null;
-  if (enableHandlers) {
-    bus = registerDefaultHandlers(store, { browser: () => ctx.send });
-  }
+  const bus = enableHandlers
+    ? registerDefaultHandlers(store, { browser: () => ctx.send })
+    : null;
   const { wsHandle, signaling } = attachSockets(server, ctx, store, {
     enableSync,
     node,
@@ -903,6 +906,7 @@ export const startServer = async ({
         ctx.closeAutomations().catch(() => {});
         ctx.autoImport?.close();
         ctx.whatsappSignIn?.stop().catch(() => {});
+        ctx.stopPulls?.();
         ctx.chatIndex.close();
         stopAll(telegramAccount, ctx.telegramAccounts, ctx.telegramExtras);
         wsHandle?.close();

package/js/src/server/routes-pull.js

@@ -0,0 +1,147 @@
+/**
+ * Inbox pulls for the networks that have no live connection of their own
+ * (R-Z8): the Connections card's "Fetch now" button and the background
+ * schedule behind it.
+ *
+ * Its routes:
+ *   POST /api/pull {source}   -> { ok, source, imported, lastOkAt }
+ *                                 or { ok: false, error, code } with `code` one of
+ *                                 unsupported | not-connected | rate-limited
+ *                                 (HTTP 429, at most one fetch per 15 s per
+ *                                 network, `retryAfterMs`) | the network's own code
+ *   GET  /api/pull/status     -> { ok, sources: { <source>: { connected,
+ *                                 lastOkAt, lastError } } }
+ *
+ * The same `createPullRunner` (../sources/pull-runner.js) serves every
+ * network: add a job to `pullJobs` to give another network a button and a
+ * schedule. VK also keeps a Long Poll running (../sources/vk-sync.js).
+ */
+
+import { createPullRunner, loadPullState } from '../sources/pull-runner.js';
+import { createVkInbox } from '../sources/vk-sync.js';
+import { liveOptionsFrom, pullLiveInto } from '../sources/index.js';
+import { createUpworkLive } from '../sources/upwork.js';
+import { createPeoplePerHourLive } from '../sources/peopleperhour.js';
+import { fail, json, readBody } from './util.js';
+
+const MINUTE = 60_000;
+
+const connected = (store, source) => async () =>
+  Boolean((await liveOptionsFrom(store, source)).token);
+
+// `live` for a network whose test double or host is given; the saved token
+// still comes from the store.
+const liveOf = (make, given) => (given ? make(given) : undefined);
+
+/** The jobs: network -> { ready, run, intervalMs }. */
+export const pullJobs = (store, { vkInbox, options = {} }) => ({
+  vk: {
+    ready: vkInbox.ready,
+    run: vkInbox.fetch,
+    intervalMs: 10 * MINUTE,
+    firstDelayMs: 30_000,
+  },
+  upwork: {
+    ready: connected(store, 'upwork'),
+    run: () =>
+      pullLiveInto(store, 'upwork', {
+        live: liveOf(createUpworkLive, options.upwork),
+      }),
+    intervalMs: 5 * MINUTE,
+    firstDelayMs: 30_000,
+  },
+  peopleperhour: {
+    ready: connected(store, 'peopleperhour'),
+    run: () =>
+      pullLiveInto(store, 'peopleperhour', {
+        live: liveOf(createPeoplePerHourLive, options.peopleperhour),
+      }),
+    intervalMs: 5 * MINUTE,
+    firstDelayMs: 30_000,
+  },
+});
+
+/**
+ * The pull services for a server: `{ pullRunner, vkInbox, startAll, stopAll }`.
+ * `options` (tests): `{ vk: {baseUrl, fetchImpl, wait}, upwork: {baseUrl},
+ * peopleperhour: {baseUrl}, minGapMs, intervals }`; `false` turns it all off.
+ */
+export const createPullServices = (store, log, options = {}) => {
+  const vkInbox = createVkInbox({ store, log, ...options.vk });
+  const jobs = pullJobs(store, { vkInbox, options });
+  for (const [source, intervalMs] of Object.entries(options.intervals ?? {})) {
+    jobs[source].intervalMs = intervalMs;
+    jobs[source].firstDelayMs = intervalMs;
+  }
+  const pullRunner = createPullRunner({
+    store,
+    jobs,
+    log,
+    minGapMs: options.minGapMs,
+  });
+  return {
+    pullRunner,
+    vkInbox,
+    start: () => {
+      pullRunner.start();
+      vkInbox.start();
+    },
+    stop: () => {
+      pullRunner.stop();
+      vkInbox.stop();
+    },
+  };
+};
+
+const statusOf = async (store, ctx) => {
+  const sources = {};
+  for (const source of ctx.pullRunner.sources()) {
+    const state = await loadPullState(store, source);
+    sources[source] = {
+      connected: await ctx.pullRunner.connected(source),
+      lastOkAt: state.lastOkAt ?? null,
+      lastError: state.lastError ?? null,
+    };
+  }
+  return { ok: true, sources };
+};
+
+const sourceOf = (body) =>
+  typeof body.source === 'string' && /^[a-z]{2,30}$/.test(body.source)
+    ? body.source
+    : null;
+
+export const handlePullRoutes = async (store, req, res, p, ctx) => {
+  if (!ctx.pullRunner) {
+    return false;
+  }
+  if (p === '/api/pull/status' && req.method === 'GET') {
+    return json(res, 200, await statusOf(store, ctx)) ?? true;
+  }
+  if (p !== '/api/pull' || req.method !== 'POST') {
+    return false;
+  }
+  const source = sourceOf(await readBody(req));
+  if (!source) {
+    return fail(res, 400, 'bad-request', 'a "source" is needed') ?? true;
+  }
+  const outcome = await ctx.pullRunner.fetchNow(source);
+  if (outcome.ok) {
+    return json(res, 200, outcome) ?? true;
+  }
+  const status =
+    { 'rate-limited': 429, unsupported: 404, 'not-connected': 409 }[
+      outcome.code
+    ] ?? 502;
+  return json(res, status, outcome) ?? true;
+};
+
+/** What a server keeps for the pulls; they start with the other `later` services. */
+export const pullContext = (store, log, options, later) => {
+  if (options === false) {
+    return { pullRunner: null };
+  }
+  const services = createPullServices(store, log, options ?? {});
+  later.push(services.start);
+  return { pullRunner: services.pullRunner, stopPulls: services.stop };
+};

package/js/src/sources/auto-import/files.js

@@ -25,7 +25,11 @@ const toArchive = (recipe, name, text, context = {}) => {
 };
 
 const parseEntry = async (provider, recipe, name, data, context) => {
-  const archive = toArchive(recipe, name, data.toString('utf8'), context);
+  // A recipe may decode its own bytes (VK's Windows-1251 pages, R-Z8).
+  const text = recipe.decode
+    ? recipe.decode(name, data)
+    : data.toString('utf8');
+  const archive = toArchive(recipe, name, text, context);
   return (await getSource(provider).parseArchive(archive)) ?? [];
 };
 

package/js/src/sources/auto-import/recipes.js

@@ -23,6 +23,11 @@
 
 import { parseCsv } from './archive.js';
 import { chatNameFromFile } from '../whatsapp-ids.js';
+import {
+  decodeVkHtml,
+  parseVkHtmlPage,
+  peerFromArchivePath,
+} from '../vk-html-archive.js';
 
 // X archives ship data as `window.YTD.<name>.part0 = [ ... ]`.
 const ytdPayload = (text) => JSON.parse(text.replace(/^[^=[]*=\s*/, ''));
@@ -64,6 +69,14 @@ const linkedinToArchive = (_name, text) => ({
   ),
 });
 
+const vkDecode = (name, bytes) =>
+  /\.html$/i.test(name) ? decodeVkHtml(bytes) : bytes.toString('utf8');
+
+const vkToArchive = (name, text) =>
+  /\.html$/i.test(name)
+    ? parseVkHtmlPage(text, { peer: peerFromArchivePath(name) })
+    : JSON.parse(text);
+
 export const autoImportRecipes = {
   email: {
     url: 'https://takeout.google.com/settings/takeout/custom/gmail',
@@ -87,7 +100,11 @@ export const autoImportRecipes = {
   vk: {
     url: 'https://vk.com/data_protection?section=rules',
     domains: ['vk.com', 'vk.ru'],
-    files: /messages.*\.json$/i,
+    // The JSON of konard/vk-export, or the pages of VK's own archive:
+    // messages/<peer>/messages<N>.html in Windows-1251 (R-Z8).
+    files: /(messages.*\.json|messages\/[^/]+\/messages\d*\.html)$/i,
+    decode: vkDecode,
+    toArchive: vkToArchive,
   },
   x: {
     url: 'https://x.com/settings/download_your_data',

package/js/src/sources/pull-runner.js

@@ -0,0 +1,171 @@
+/**
+ * One small runner for "fetch now" and "fetch on a schedule" of a network's
+ * inbox (R-Z8; the page's Fetch button and the background poller share it).
+ *
+ * A job is `{ ready(), run(), intervalMs }`:
+ *  - `ready()` is true when the network is connected (a saved token), so a
+ *    network nobody connected is never polled;
+ *  - `run()` pulls and stores, returning `{ imported, ... }`. It saves any
+ *    cursor itself, and only AFTER the store write succeeded, so a failed
+ *    write is read again next time instead of skipped.
+ *
+ * `fetchNow(source)` is the button: at most one run per `minGapMs` (15 s) for
+ * a network, background runs counted too, and one at a time. `start()` is
+ * the schedule: each ready job runs every `intervalMs`, and after a failure
+ * the wait doubles up to `maxBackoffMs`, so a dead token or a down network
+ * is not hammered. Nothing starts before `start()`, which the server calls
+ * only once it listens. Errors come back as `{ ok: false, error, code }`
+ * with secrets masked; nothing here throws at a caller.
+ *
+ * `pull-state:<source>` keeps `lastOkAt` and a job's own cursor.
+ */
+
+import { safeMessage } from '../server/redact.js';
+
+export const PULL_MIN_GAP_MS = 15_000;
+const MAX_BACKOFF_MS = 60 * 60_000;
+
+const stateId = (source) => `pull-state:${source}`;
+// Writes of one source's state are queued, so two patches never overwrite
+// each other (a manual fetch and the poller can overlap).
+const queues = new WeakMap();
+
+export const loadPullState = async (store, source) =>
+  (await store.get(stateId(source))) ?? {};
+
+/** Merge `patch` into the source's state link; the write has finished on return. */
+export const savePullState = (store, source, patch) => {
+  const byStore = queues.get(store) ?? new Map();
+  queues.set(store, byStore);
+  const run = async () => {
+    const current = await loadPullState(store, source);
+    const next = {
+      ...current,
+      ...patch,
+      id: stateId(source),
+      tokens: ['pull', 'state', source],
+      children: [],
+    };
+    await store.put(next);
+    return next;
+  };
+  const queued = (byStore.get(source) ?? Promise.resolve()).then(run, run);
+  byStore.set(
+    source,
+    queued.catch(() => {})
+  );
+  return queued;
+};
+
+const failure = (error, extra = {}) => ({
+  ok: false,
+  error: safeMessage(error),
+  code: error?.code && typeof error.code === 'string' ? error.code : 'failed',
+  ...extra,
+});
+
+export const createPullRunner = ({
+  store,
+  jobs,
+  now = Date.now,
+  minGapMs = PULL_MIN_GAP_MS,
+  maxBackoffMs = MAX_BACKOFF_MS,
+  log = null,
+  setTimer = globalThis.setTimeout,
+  clearTimer = globalThis.clearTimeout,
+} = {}) => {
+  const lastStart = new Map();
+  const running = new Set();
+  const timers = new Map();
+  let stopped = true;
+
+  const execute = async (source) => {
+    const job = jobs[source];
+    running.add(source);
+    lastStart.set(source, now());
+    try {
+      const result = (await job.run()) ?? {};
+      const lastOkAt = new Date(now()).toISOString();
+      await savePullState(store, source, { lastOkAt, lastError: null });
+      return { ok: true, source, lastOkAt, imported: result.imported ?? 0 };
+    } catch (error) {
+      const out = failure(error, { source });
+      log?.({ level: 'warn', event: 'pull_failed', source, code: out.code });
+      await savePullState(store, source, {
+        lastError: { code: out.code, at: new Date(now()).toISOString() },
+      }).catch(() => {});
+      return out;
+    } finally {
+      running.delete(source);
+    }
+  };
+
+  const fetchNow = async (source) => {
+    const job = jobs[source];
+    if (!job) {
+      return failure(new Error(`no inbox pull for "${source}"`), {
+        code: 'unsupported',
+      });
+    }
+    if (!(await job.ready())) {
+      return failure(new Error(`${source} is not connected: save its token`), {
+        code: 'not-connected',
+      });
+    }
+    const since = now() - (lastStart.get(source) ?? -Infinity);
+    if (running.has(source) || since < minGapMs) {
+      return failure(new Error('fetched a moment ago; wait a few seconds'), {
+        code: 'rate-limited',
+        retryAfterMs: running.has(source) ? minGapMs : minGapMs - since,
+      });
+    }
+    return execute(source);
+  };
+
+  // One scheduled run; the number of failures in a row after it.
+  const tick = async (source, fails) => {
+    try {
+      if (!(await jobs[source].ready()) || running.has(source)) {
+        return fails;
+      }
+      return (await execute(source)).ok ? 0 : fails + 1;
+    } catch {
+      return fails + 1;
+    }
+  };
+
+  const schedule = (source, delay, fails) => {
+    if (stopped) {
+      return;
+    }
+    const timer = setTimer(async () => {
+      const next = await tick(source, fails);
+      const wait = jobs[source].intervalMs * 2 ** Math.min(next, 10);
+      schedule(source, Math.min(wait, maxBackoffMs), next);
+    }, delay);
+    timer?.unref?.();
+    timers.set(source, timer);
+  };
+
+  return {
+    fetchNow,
+    /** Start the schedules (call after the server listens). */
+    start() {
+      stopped = false;
+      for (const [source, job] of Object.entries(jobs)) {
+        if (job.intervalMs > 0) {
+          schedule(source, job.firstDelayMs ?? job.intervalMs, 0);
+        }
+      }
+    },
+    stop() {
+      stopped = true;
+      for (const timer of timers.values()) {
+        clearTimer(timer);
+      }
+      timers.clear();
+    },
+    sources: () => Object.keys(jobs),
+    connected: (source) => jobs[source].ready(),
+  };
+};