@keepsnappy/react-native 0.1.0-alpha.15 → 0.1.0-alpha.17

package/js/sdk/push.ts

@@ -11,6 +11,8 @@
 // rewritten to the live queue whenever it outgrows twice the queue caps, so it
 // stays bounded while the ingest is unreachable.
 
+import { MAX_BATCH_BYTES, utf8Length } from './bytes';
+
 export interface DeviceInfo {
   model: string | null;
   os: string | null;
@@ -113,6 +115,8 @@ const serverClock = async (res: { json?: () => Promise<unknown> }, bound: Promis
 interface Item {
   line: string;
   bytes: number;
+  // The line's size inside the request body: a JSON string (escaped) plus its comma.
+  wire: number;
   token: string | null;
   // The process that recorded the line (its bootId) and its sequence there.
   origin: string;
@@ -125,7 +129,7 @@ interface Batch {
   id: string;
   items: Item[];
   token: string | null;
-  bytes: number;
+  wire: number;
 }
 
 interface SpoolAdd {
@@ -208,31 +212,15 @@ export const readSpool = (content: string | null): { pending: PendingLine[]; dro
 export const pendingFromSpool = (content: string | null): PendingLine[] =>
   readSpool(content).pending;
 
-const utf8Length = (text: string) => {
-  let n = 0;
-  for (let i = 0; i < text.length; i++) {
-    const c = text.charCodeAt(i);
-    if (c < 0x80) {
-      n += 1;
-    } else if (c < 0x800) {
-      n += 2;
-    } else if (c >= 0xd800 && c <= 0xdbff) {
-      n += 4;
-      i++;
-    } else {
-      n += 3;
-    }
-  }
-  return n;
-};
-
 // Retrying cannot fix these; the batch is dropped instead of blocking the queue.
+const wireBytes = (line: string) => utf8Length(JSON.stringify(line)) + 1;
+
 const isPermanent = (status: number) =>
   status >= 400 && status < 500 && status !== 408 && status !== 429;
 
 export const createPushClient = (deps: PushDeps) => {
   const idleFlushMs = deps.idleFlushMs ?? 10_000;
-  const maxBatchBytes = deps.maxBatchBytes ?? 256 * 1024;
+  const maxBatchBytes = deps.maxBatchBytes ?? MAX_BATCH_BYTES;
   const maxBatchItems = deps.maxBatchItems ?? 400;
   const maxQueueLines = deps.maxQueueLines ?? 2000;
   const maxQueueBytes = deps.maxQueueBytes ?? 8 * 1024 * 1024;
@@ -277,6 +265,11 @@ export const createPushClient = (deps: PushDeps) => {
   let journalRecords = 0;
   // Lines dropped before delivery, by this process and (from the note) earlier ones.
   let undelivered = 0;
+  // Lines of a run dropped before delivery (queue cap, line over the batch limit), by run token, until its next flush.
+  const droppedOfRun = new Map<string, number>();
+  const noteDropped = (token: string | null) => {
+    if (token !== null) droppedOfRun.set(token, (droppedOfRun.get(token) ?? 0) + 1);
+  };
   // Round trips not reported yet, each with the run token it was measured under.
   let clockPending: { token: string; sample: ClockSample }[] = [];
   const clockOf = (token: string | null) =>
@@ -440,6 +433,7 @@ export const createPushClient = (deps: PushDeps) => {
       }
     }
     undelivered += dropped.length;
+    for (const item of dropped) noteDropped(item.token);
     return dropped;
   };
 
@@ -448,12 +442,14 @@ export const createPushClient = (deps: PushDeps) => {
     if (bytes > maxBatchBytes) {
       stats.dropped++;
       undelivered++;
+      noteDropped(deps.getRunToken());
       stats.lastError = `line of ${bytes} bytes exceeds batch limit`;
       return;
     }
     const item: Item = {
       line,
       bytes,
+      wire: wireBytes(line),
       token: deps.getRunToken(),
       origin: bootId,
       seq: nextSeq++,
@@ -502,11 +498,13 @@ export const createPushClient = (deps: PushDeps) => {
       if (bytes > maxBatchBytes) {
         stats.resumeDropped++;
         undelivered++;
+        noteDropped(record.t);
         continue;
       }
       earlier.push({
         line: record.l,
         bytes,
+        wire: wireBytes(record.l),
         token: record.t,
         origin: record.o,
         seq: record.s,
@@ -544,16 +542,16 @@ export const createPushClient = (deps: PushDeps) => {
     }
     const first = queue[0] as Item;
     const items: Item[] = [];
-    let bytes = 0;
+    let wire = 0;
     if (first.batch !== undefined) {
       // Already sent once (by a killed process): resend exactly that batch.
       while (queue.length > 0 && queue[0]?.batch === first.batch) {
         const next = queue.shift() as Item;
         queuedBytes -= next.bytes;
         items.push(next);
-        bytes += next.bytes;
+        wire += next.wire;
       }
-      return { id: first.batch, items, token: first.token, bytes };
+      return { id: first.batch, items, token: first.token, wire };
     }
     while (queue.length > 0 && items.length < maxBatchItems) {
       const next = queue[0];
@@ -562,16 +560,16 @@ export const createPushClient = (deps: PushDeps) => {
         next.batch !== undefined ||
         next.token !== first.token ||
         next.origin !== first.origin ||
-        (items.length > 0 && bytes + next.bytes > maxBatchBytes)
+        (items.length > 0 && wire + next.wire > maxBatchBytes)
       ) {
         break;
       }
       queue.shift();
       queuedBytes -= next.bytes;
       items.push(next);
-      bytes += next.bytes;
+      wire += next.wire;
     }
-    return { id: '', items, token: first.token, bytes };
+    return { id: '', items, token: first.token, wire };
   };
 
   let inflight: Promise<void> | null = null;
@@ -617,29 +615,48 @@ export const createPushClient = (deps: PushDeps) => {
           bound.clear();
         }
       }
+      const runToken = batch.token;
+      const reported = clockOf(runToken);
+      const sentDevice = device;
+      const bodyOf = (id: string) => JSON.stringify({
+        batch_id: id,
+        device: sentDevice,
+        items: batch.items.map(item => item.line),
+        ...(runToken ? { run_token: runToken } : {}),
+        ...(reported.length ? { clock: reported } : {}),
+      });
+      // Names exactly these lines (process, first and last sequence, count),
+      // so a different set of lines is always a different batch.
+      const idOf = () => {
+        const first = batch.items[0] as Item;
+        const last = batch.items[batch.items.length - 1] as Item;
+        return `${sentDevice.install_id}-${first.origin}-${first.seq}-${last.seq}-${batch.items.length}`;
+      };
+      let body = bodyOf(batch.id || idOf());
       if (!batch.id) {
-        // Names exactly these lines (process, first and last sequence, count),
-        // so a different set of lines is always a different batch. It is
-        // journalled before the first send: a later process that resends after
+        // The cap is on the body as sent (escaped lines, device, clock samples): lines over it go back to the queue
+        // front for the next batch. A lone line still goes; a line alone over the cap was dropped at enqueue.
+        for (let over = utf8Length(body) - maxBatchBytes; over > 0 && batch.items.length > 1; over = utf8Length(body) - maxBatchBytes) {
+          const back: Item[] = [];
+          for (let freed = 0; freed < over && batch.items.length > 1;) {
+            const item = batch.items.pop() as Item;
+            freed += item.wire;
+            back.unshift(item);
+          }
+          queue.unshift(...back);
+          for (const item of back) queuedBytes += item.bytes;
+          body = bodyOf(idOf());
+        }
+        // Journalled before the first send: a later process that resends after
         // a kill sends the same lines under the same id, and the server's
         // replay check can only ever drop lines it already has.
+        batch.id = idOf();
         const first = batch.items[0] as Item;
-        const last = batch.items[batch.items.length - 1] as Item;
-        batch.id = `${device.install_id}-${first.origin}-${first.seq}-${last.seq}-${batch.items.length}`;
         for (const item of batch.items) {
           item.batch = batch.id;
         }
         journal({ b: batch.id, o: first.origin, s: batch.items.map(item => item.seq) });
       }
-      const runToken = batch.token;
-      const reported = clockOf(runToken);
-      const body = JSON.stringify({
-        batch_id: batch.id,
-        device,
-        items: batch.items.map(item => item.line),
-        ...(runToken ? { run_token: runToken } : {}),
-        ...(reported.length ? { clock: reported } : {}),
-      });
       const headers: Record<string, string> = {
         'content-type': 'application/json',
         'x-keepsnappy-key': deps.key,
@@ -820,16 +837,23 @@ export const createPushClient = (deps: PushDeps) => {
         await sendClock(deadline);
       }
       const refused = refusedRunBatches - refusedBefore;
-      const ok = sentAll && refused === 0;
+      // A run that lost lines before they were sent is missing windows, whatever went up after.
+      const token = deps.getRunToken();
+      const lost = token === null ? 0 : (droppedOfRun.get(token) ?? 0);
+      if (token !== null) droppedOfRun.delete(token);
+      const ok = sentAll && refused === 0 && lost === 0;
+      const error = ok
+        ? null
+        : !sentAll
+          ? (stats.lastError ?? 'flush deadline')
+          : refused > 0
+            ? `rejected ${lastRefusedStatus}: ${refused} ${refused === 1 ? 'batch' : 'batches'} of the run dropped`
+            : 'lines of the run dropped before sending';
       return {
         ok,
         batches: stats.batchesSent - sentBefore,
         lines: stats.linesSent - linesBefore,
-        error: ok
-          ? null
-          : !sentAll
-            ? (stats.lastError ?? 'flush deadline')
-            : `rejected ${lastRefusedStatus}: ${refused} ${refused === 1 ? 'batch' : 'batches'} of the run dropped`,
+        error: error !== null && lost > 0 ? `${error} dropped=${lost}` : error,
       };
     } finally {
       flushLimit = null;

package/js/sdk/sampler.ts

@@ -132,7 +132,7 @@ export const estimateOffset = (before: number, native: number, after: number) =>
 // late after its t0 a window can start and still take one. It gates only screen windows (a screen window opened by a
 // press is detected after the new screen rendered and takes the press's open sample, taken the touch delivery delay
 // past the press's t0) and the reuse of a close sample for a restarted key. Press windows do not depend on it: they
-// take the previous window's close sample by contiguity. Evidence for 50: idle lab event timing of 4.9-19.1 ms
+// take the previous window's close sample by contiguity, else a sample of their own however late it is. Evidence for 50: idle lab event timing of 4.9-19.1 ms
 // (packages/derive/test/fixtures/p37-*.log); no loaded-device data. Measure the touchstart delivery delay p99 per
 // device from farm data (docs/log/followups.md).
 export const OPEN_SAMPLE_TOLERANCE_MS = 50;

package/js/sdk/screens.ts

@@ -26,7 +26,7 @@ import {
 } from './core';
 import { retainFloorOf } from './retain-floor';
 import { type CauseRoot, currentCause, latestActionAt, launchedAt, launchRootId } from './causes';
-import { routeOfScreenHost, routes } from './fiber';
+import { buildPlaceId, routeOfScreenHost, routes, screenKeyForRoute, setScreenKeyProvider, currentFiber, type ScreenRouteResolution, type Fiber } from './fiber';
 import {
   lastPressStart,
   type NativeScreenEvent,
@@ -64,6 +64,12 @@ interface Visit {
   chainKey: string;
   keyChain: string;
   visit: number;
+  screenKey: string | null;
+  legacyName: string;
+  // The screen component is not known yet: its scene has not mounted, or it is suspended.
+  pending: boolean;
+  resolution: ScreenRouteResolution;
+  lazyBoundary: Fiber | null;
   t0: number;
   openedAt: number;
   endedAt: number | null;
@@ -89,6 +95,7 @@ interface Resolution {
 let current: Visit | null = null;
 let seq = 0;
 const visitsByChain = new Map<string, number>();
+const visitsByScreen = new Map<string, number>();
 let lastActionAt = -Infinity;
 interface AppLink {
   t: number;
@@ -248,6 +255,7 @@ const endVisit = (reason: string, at?: number) => {
   if (!visit || visit.endedAt !== null) {
     return;
   }
+  if (visit.pending) resolveVisit(visit, true);
   const t = Math.max(visit.t0, at ?? now());
   visit.endedAt = t;
   stats.ends[reason] = (stats.ends[reason] ?? 0) + 1;
@@ -262,7 +270,9 @@ const endVisit = (reason: string, at?: number) => {
 // state event.
 const chooseStart = (snapshot: NavSnapshot, floor: number) => {
   const inSpan = (t: number) => t >= floor && t <= snapshot.stateT;
-  const presses = pressesSinceVisit.filter(p => inSpan(p.t0));
+  // Touches delivered since the previous visit opened, whatever their finger time: a finger that landed while the
+  // previous screen was mounting reaches JS after its state was read.
+  const presses = pressesSinceVisit.filter(p => p.t0 >= windowsEndedAt && p.t0 <= snapshot.stateT);
   const link = pendingLink && inSpan(pendingLink.t) ? pendingLink : null;
   const action = snapshot.action && inSpan(snapshot.action.t) ? snapshot.action : null;
   const touchStart = (p: Press) => ({
@@ -332,6 +342,28 @@ const isStrictExtension = (prev: RoutePart[], next: RoutePart[]) =>
       (part.key ?? null) === (next[i]?.key ?? null),
   );
 
+const screenResolution = (chain: RoutePart[]) => {
+  const leaf = chain[chain.length - 1]!;
+  return screenKeyForRoute(leaf.key ?? null, leaf.name);
+};
+
+const moveScreenCounter = (visit: Visit, screen: string) => {
+  if (visit.screenKey === screen) return;
+  if (visit.screenKey !== null) visitsByScreen.set(visit.screenKey, (visitsByScreen.get(visit.screenKey) ?? 1) - 1);
+  visit.visit = (visitsByScreen.get(screen) ?? 0) + 1;
+  visitsByScreen.set(screen, visit.visit);
+  visit.screenKey = screen;
+};
+
+const resolveVisit = (visit: Visit, final = false) => {
+  const resolved = visit.resolution;
+  if (resolved.screenPending && !final) return;
+  moveScreenCounter(visit, resolved.screen);
+  visit.name = buildPlaceId({ kind: 'screen', screen: resolved.screen, visit: visit.visit });
+  visit.pending = false;
+  markers.rename(visit.key, visit.name, visit.legacyName);
+};
+
 const extend = (visit: Visit, chain: RoutePart[], chainKey: string) => {
   const previous = visitsByChain.get(visit.chainKey) ?? 1;
   visitsByChain.set(visit.chainKey, previous - 1);
@@ -341,9 +373,19 @@ const extend = (visit: Visit, chain: RoutePart[], chainKey: string) => {
   visit.chain = chain;
   visit.chainKey = chainKey;
   visit.keyChain = keyChainOf(chain);
-  visit.visit = n;
-  visit.name = nameOf(chainKey, n);
-  markers.rename(visit.key, visit.name);
+  visit.legacyName = nameOf(chainKey, n);
+  const resolution = screenResolution(chain);
+  visit.resolution = resolution;
+  visit.lazyBoundary = resolution.suspended ?? null;
+  if (resolution.screenPending) {
+    if (visit.screenKey !== null) {
+      visitsByScreen.set(visit.screenKey, (visitsByScreen.get(visit.screenKey) ?? 1) - 1);
+      visit.screenKey = null;
+    }
+    visit.name = visit.legacyName;
+    visit.pending = true;
+    markers.rename(visit.key, visit.name, visit.legacyName);
+  } else resolveVisit(visit);
   stats.extended++;
 };
 
@@ -398,6 +440,7 @@ const onSnapshot = (snapshot: NavSnapshot, nameSource: NameSource) => {
   const n = (visitsByChain.get(chainKey) ?? 0) + 1;
   visitsByChain.set(chainKey, n);
   seq++;
+  const resolved = screenResolution(chain);
   const visit: Visit = {
     key: `screen#${seq}`,
     name: nameOf(chainKey, n),
@@ -405,6 +448,11 @@ const onSnapshot = (snapshot: NavSnapshot, nameSource: NameSource) => {
     chainKey,
     keyChain,
     visit: n,
+    legacyName: nameOf(chainKey, n),
+    screenKey: null,
+    pending: true,
+    resolution: resolved,
+    lazyBoundary: resolved.suspended ?? null,
     t0: start.t0,
     openedAt: detectedAt,
     endedAt: null,
@@ -417,6 +465,11 @@ const onSnapshot = (snapshot: NavSnapshot, nameSource: NameSource) => {
     launch,
     cause: snapshot.cause,
   };
+  if (!resolved.screenPending) {
+    moveScreenCounter(visit, resolved.screen);
+    visit.name = buildPlaceId({ kind: 'screen', screen: resolved.screen, visit: visit.visit });
+    visit.pending = false;
+  }
   current = visit;
   pressesSinceVisit = [];
   stats.visits++;
@@ -429,6 +482,12 @@ const onSnapshot = (snapshot: NavSnapshot, nameSource: NameSource) => {
   markers.start(visit.name, {
     key: visit.key,
     kind: 'screen',
+    legacyName: visit.legacyName,
+    beforeEnd: () => {
+      if (visit.pending) resolveVisit(visit, true);
+      visit.lazyBoundary = null;
+      visit.resolution = { screen: visit.screenKey ?? visit.resolution.screen };
+    },
     at: visit.t0,
     details: details(visit),
     // The launch window belongs to the launch, whatever action came before the first state (REQ-MEAS-22).
@@ -470,12 +529,14 @@ export const noteAppLink = (url: string, via: string) => {
   }
 };
 
+// Every touch delivered while a visit is open is an action after the one that opened it, even when its finger landed
+// before the SDK read the navigation state (the JS thread was busy with the mount): order decides, not the clocks.
 const onPress = (t0: number) => {
   const press = lastPressStart();
   if (press) {
     pressesSinceVisit.push(press);
   }
-  if (current && current.endedAt === null && t0 > current.openedAt) {
+  if (current && current.endedAt === null) {
     endVisit('next-action', t0);
   }
 };
@@ -530,6 +591,7 @@ export const screenStats = () => ({
   visitsByChain: Object.fromEntries(
     [...visitsByChain].filter(([, n]) => n > 0).slice(0, MAX_LISTED),
   ),
+  visitsByScreen: Object.fromEntries([...visitsByScreen].filter(([, n]) => n > 0).slice(0, MAX_LISTED)),
   trackedRoutes: routes.size,
   nav: navStats(),
   native: {
@@ -569,6 +631,10 @@ export const installScreens = () => {
     return;
   }
   installed = true;
+  setScreenKeyProvider(() => current ? {
+    screen: current.screenKey ?? current.resolution.screen,
+    ...(current.pending || current.lazyBoundary ? { screenPending: true } : {}),
+  } : { screen: '(no screen)' });
   setRetainFloor(screenRetainFloor);
   onWindowsEnd(onWindowsEnded);
   onNavSnapshot(snapshot => onSnapshot(snapshot, 'state'));
@@ -578,6 +644,25 @@ export const installScreens = () => {
     setTimeout(() => logNativeEvent(event), RESOLVE_DELAY_MS);
   });
   onAfterCommit(() => {
+    const visit = current;
+    if (visit && isWindowOpen(visit.key)) {
+      // A pending visit is resolved when its scene mounts, however many commits come first; it falls back to the route
+      // name only when it ends.
+      if (visit.pending && !visit.lazyBoundary) {
+        visit.resolution = screenResolution(visit.chain);
+        visit.lazyBoundary = visit.resolution.suspended ?? null;
+        resolveVisit(visit);
+      } else if (visit.lazyBoundary) {
+        const boundary = currentFiber(visit.lazyBoundary);
+        if (!boundary || boundary.memoizedState == null) {
+          visit.lazyBoundary = null;
+          if (boundary) {
+            visit.resolution = screenResolution(visit.chain);
+            resolveVisit(visit, true);
+          }
+        }
+      }
+    }
     if (!checkScheduled && containerCount() === 0) {
       checkScheduled = true;
       setTimeout(checkRouteProps, 0);

package/js/sdk/stream.ts

@@ -39,6 +39,12 @@ export const createStream = (processId: string, maxFibers = MAX_SEGMENT_FIBERS)
     /** The segment the next commit goes into: windows name it so the backend can tell a stream is complete. */
     seq: () => seq,
     size: () => commits.length,
+    /** Sets the fiber cap of the segments from now on; returns the previous cap. */
+    setMaxFibers(n: number) {
+      const previous = maxFibers;
+      maxFibers = n;
+      return previous;
+    },
     noteRoot(root: CauseRoot) {
       roots.push(root);
     },

package/metro.js

@@ -175,7 +175,7 @@ const resolveEntry = projectRoot => {
   return null;
 };
 
-function withKeepSnappy(input) {
+function withKeepSnappy(input, { clientPackages = [] } = {}) {
   if (!isEnabled()) {
     return isProfiling() ? withReactProfiling(input) : input;
   }
@@ -201,13 +201,23 @@ function withKeepSnappy(input) {
   process.env[ENV_UPSTREAM] = upstream;
   process.env[ENV_ENTRY] = entry;
   process.env[ENV_BOOT] = BOOT_PATH;
+  const packages = [...new Set(clientPackages)].sort();
+  process.env.KEEPSNAPPY_CLIENT_PACKAGES = JSON.stringify(packages);
+  process.env.KEEPSNAPPY_PROJECT_ROOT = projectRoot;
+  process.env.KEEPSNAPPY_COMPONENT_CACHE_KEY = require('./js/component-stamps.js').configure(projectRoot, packages);
 
   const serializer = config.serializer ?? {};
   const upstreamPolyfills = serializer.getPolyfills;
   const upstreamPreMain = serializer.getModulesRunBeforeMainModule;
+  const resolver = config.resolver ?? {};
+  const blockList = resolver.blockList ? (Array.isArray(resolver.blockList) ? resolver.blockList : [resolver.blockList]) : [];
+  const escape = value => value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+  const roots = [...new Set([path.resolve(projectRoot), fs.realpathSync(projectRoot)])];
+  const inventory = new RegExp(`^(?:${roots.map(escape).join('|')})[/\\\\]\\.keepsnappy(?:[/\\\\]|$)`);
 
   return {
     ...config,
+    resolver: { ...resolver, blockList: [...blockList, inventory] },
     serializer: {
       ...serializer,
       getPolyfills: options => [

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@keepsnappy/react-native",
-  "version": "0.1.0-alpha.15",
+  "version": "0.1.0-alpha.17",
   "description": "KeepSnappy in-app SDK for React Native / Expo: zero-code measurement windows, React commit hook, native signposts and log file",
   "license": "UNLICENSED",
   "repository": {