@lowdefy/server-dev 6.0.0 → 7.0.0

package/client/recorder/buildTraceEvent.js

@@ -0,0 +1,57 @@
+/*
+  Copyright 2020-2026 Lowdefy, Inc
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+import { type } from '@lowdefy/helpers';
+
+import collectEventActions from './collectEventActions.js';
+import collectEventEndpoints from './collectEventEndpoints.js';
+import collectEventRequests from './collectEventRequests.js';
+import diffStateWrites from './diffStateWrites.js';
+
+// One completed engine event as a trace record's `event` (or `also` entry),
+// built from the engine's trace payload the moment it arrives: the state the
+// event wrote is the difference between the copy taken before its actions ran
+// and the state now, which later events would change.
+function buildTraceEvent({ payload, urlAfter }) {
+  const { context, failure, record } = payload;
+  const responses = record?.responses ?? {};
+  const event = {
+    name: payload.eventName,
+    block_id: payload.blockId,
+    success: payload.success,
+    actions: collectEventActions({ responses }),
+    requests: collectEventRequests({ blockId: payload.blockId, context, responses }),
+    endpoints: collectEventEndpoints({ blockId: payload.blockId, context, responses }),
+    state_writes: diffStateWrites({
+      before: payload.stateBefore,
+      after: context?.state,
+      values: true,
+    }),
+    url_after: urlAfter ?? null,
+  };
+  if (!type.isNone(failure)) {
+    event.error = {
+      name: failure.errorName ?? 'Error',
+      config_key: failure.configKey ?? null,
+      action_type: failure.actionType ?? null,
+      action_id: failure.actionId ?? null,
+    };
+    event.invalid_blocks = failure.invalidBlocks ?? [];
+  }
+  return event;
+}
+
+export default buildTraceEvent;

package/client/recorder/buildTraceEvent.test.mjs

@@ -0,0 +1,81 @@
+/*
+  Copyright 2020-2026 Lowdefy, Inc
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+import buildTraceEvent from './buildTraceEvent.js';
+
+function payload({ responses, requests = {}, apiResponses = {} }) {
+  return {
+    eventName: 'onClick',
+    blockId: 'save',
+    success: true,
+    failure: null,
+    stateBefore: { a: 1 },
+    record: { responses },
+    context: { state: { a: 1 }, requests, _internal: { lowdefy: { apiResponses } } },
+  };
+}
+
+test('buildTraceEvent lists the actions the event ran, in order, without skipped ones', () => {
+  const event = buildTraceEvent({
+    payload: payload({
+      responses: {
+        link: { type: 'Link', index: 2 },
+        validate: { type: 'Validate', index: 0 },
+        skipped: { type: 'SetState', index: 1, skipped: true },
+      },
+    }),
+  });
+  expect(event.actions).toEqual(['Validate', 'Link']);
+});
+
+test('buildTraceEvent names the endpoints its CallAPI actions called', () => {
+  const event = buildTraceEvent({
+    payload: payload({
+      responses: { call: { type: 'CallAPI', index: 0 } },
+      apiResponses: {
+        syncInvoices: [{ actionId: 'call', blockId: 'save', success: true, responseTime: 30 }],
+        failedElsewhere: [{ actionId: 'other', blockId: 'save', success: false, responseTime: 5 }],
+        refused: [{ actionId: 'call', blockId: 'save', success: false }],
+      },
+    }),
+  });
+  expect(event.endpoints).toEqual([
+    { id: 'syncInvoices', ok: true, ms: 30 },
+    { id: 'refused', ok: false, ms: null },
+  ]);
+});
+
+test('buildTraceEvent leaves out calls another block or a skipped action made', () => {
+  const event = buildTraceEvent({
+    payload: payload({
+      responses: {
+        call: { type: 'CallAPI', index: 0 },
+        fetch: { type: 'Request', index: 1, skipped: true },
+      },
+      requests: {
+        listItems: [{ actionId: 'call', blockId: 'refresh' }],
+        getItem: [{ actionId: 'fetch', blockId: 'save' }],
+        saveItem: [{ actionId: 'call', blockId: 'save', responseTime: 12 }],
+      },
+      apiResponses: {
+        notify: [{ actionId: 'call', blockId: 'other_button', success: true }],
+        archive: [{ actionId: 'fetch', blockId: 'save', success: true }],
+      },
+    }),
+  });
+  expect(event.requests).toEqual([{ id: 'saveItem', ok: true, ms: 12 }]);
+  expect(event.endpoints).toEqual([]);
+});

package/client/recorder/chunkRecords.js

@@ -0,0 +1,69 @@
+/*
+  Copyright 2020-2026 Lowdefy, Inc
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+// Keepalive fetches share a 64 KB in-flight budget per page, so each POST body
+// stays under 48 KB.
+const MAX_CHUNK_BYTES = 48 * 1024;
+
+function utf8Length(text) {
+  let bytes = 0;
+  for (const character of text) {
+    const code = character.codePointAt(0);
+    if (code < 0x80) {
+      bytes += 1;
+    } else if (code < 0x800) {
+      bytes += 2;
+    } else if (code < 0x10000) {
+      bytes += 3;
+    } else {
+      bytes += 4;
+    }
+  }
+  return bytes;
+}
+
+// The bytes of `{"session":…,"records":[]}` before any record is added.
+function emptyBodyLength(session) {
+  return utf8Length(JSON.stringify({ session, records: [] }));
+}
+
+// Splits records into POST bodies of { session, records }, one session per
+// body (the route names the file by it) and each body under maxBytes. A
+// single record too large for a body on its own is dropped.
+function chunkRecords({ records, maxBytes = MAX_CHUNK_BYTES }) {
+  const chunks = [];
+  let current = null;
+  records.forEach((record) => {
+    const recordLength = utf8Length(JSON.stringify(record));
+    if (current !== null && current.session === record.session) {
+      // One comma joins the record to the ones before it.
+      const length = current.length + 1 + recordLength;
+      if (length <= maxBytes) {
+        current.records.push(record);
+        current.length = length;
+        return;
+      }
+    }
+    if (current !== null) chunks.push({ session: current.session, records: current.records });
+    const length = emptyBodyLength(record.session) + recordLength;
+    current = length <= maxBytes ? { session: record.session, records: [record], length } : null;
+  });
+  if (current !== null) chunks.push({ session: current.session, records: current.records });
+  return chunks;
+}
+
+export { MAX_CHUNK_BYTES };
+export default chunkRecords;

package/client/recorder/chunkRecords.test.mjs

@@ -0,0 +1,59 @@
+/*
+  Copyright 2020-2026 Lowdefy, Inc
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+import chunkRecords, { MAX_CHUNK_BYTES } from './chunkRecords.js';
+
+function record(session, size = 1000) {
+  return { session, t: '2026-10-03T14:03:11.000Z', kind: 'change', value: 'x'.repeat(size) };
+}
+
+test('chunkRecords keeps every POST body under 48 KB', () => {
+  const records = Array.from({ length: 200 }, () => record('20261003T140311Z-aaaaaa'));
+  const chunks = chunkRecords({ records });
+  expect(chunks.length).toBeGreaterThan(1);
+  chunks.forEach((chunk) => {
+    expect(Buffer.byteLength(JSON.stringify(chunk))).toBeLessThanOrEqual(MAX_CHUNK_BYTES);
+  });
+  expect(chunks.flatMap((chunk) => chunk.records)).toHaveLength(200);
+});
+
+test('chunkRecords counts multi-byte characters by their UTF-8 length', () => {
+  const records = Array.from({ length: 40 }, () => ({
+    session: '20261003T140311Z-aaaaaa',
+    value: 'é€😀'.repeat(200),
+  }));
+  chunkRecords({ records, maxBytes: 8000 }).forEach((chunk) => {
+    expect(Buffer.byteLength(JSON.stringify(chunk))).toBeLessThanOrEqual(8000);
+  });
+});
+
+test('chunkRecords puts one session in each body', () => {
+  const chunks = chunkRecords({
+    records: [record('20261003T140311Z-aaaaaa', 10), record('20261003T142311Z-bbbbbb', 10)],
+  });
+  expect(chunks.map((chunk) => chunk.session)).toEqual([
+    '20261003T140311Z-aaaaaa',
+    '20261003T142311Z-bbbbbb',
+  ]);
+});
+
+test('chunkRecords drops a single record too large for a body', () => {
+  const chunks = chunkRecords({
+    records: [record('20261003T140311Z-aaaaaa', 60000), record('20261003T140311Z-aaaaaa', 10)],
+  });
+  expect(chunks).toHaveLength(1);
+  expect(chunks[0].records).toHaveLength(1);
+});

package/client/recorder/collectEventActions.js

@@ -0,0 +1,30 @@
+/*
+  Copyright 2020-2026 Lowdefy, Inc
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+import { type } from '@lowdefy/helpers';
+
+// The types of the actions this event ran, in the order they ran: every
+// response entry the engine kept, without the actions a skip or a control
+// passed over.
+function collectEventActions({ responses }) {
+  return Object.values(responses)
+    .filter((response) => type.isObject(response) && response.skipped !== true)
+    .filter((response) => type.isString(response.type))
+    .sort((a, b) => (a.index ?? 0) - (b.index ?? 0))
+    .map((response) => response.type);
+}
+
+export default collectEventActions;

package/client/recorder/collectEventEndpoints.js

@@ -0,0 +1,40 @@
+/*
+  Copyright 2020-2026 Lowdefy, Inc
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+import { type } from '@lowdefy/helpers';
+
+import isEventCall from './isEventCall.js';
+
+// The API endpoints this event's CallAPI actions called, as trace record
+// entries. `apiResponses[endpointId]` is a newest-first call history whose
+// entries carry the block and the id of the action that made the call, read
+// as collectEventRequests reads request calls.
+function collectEventEndpoints({ blockId, context, responses }) {
+  const apiResponses = context?._internal?.lowdefy?.apiResponses ?? {};
+  const endpoints = [];
+  Object.keys(apiResponses).forEach((endpointId) => {
+    const call = (apiResponses[endpointId] ?? [])[0];
+    if (!isEventCall({ blockId, call, responses })) return;
+    endpoints.push({
+      id: endpointId,
+      ok: call.success === true,
+      ms: type.isNumber(call.responseTime) ? call.responseTime : null,
+    });
+  });
+  return endpoints;
+}
+
+export default collectEventEndpoints;

package/client/recorder/collectEventRequests.js

@@ -0,0 +1,39 @@
+/*
+  Copyright 2020-2026 Lowdefy, Inc
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+import { type } from '@lowdefy/helpers';
+
+import isEventCall from './isEventCall.js';
+
+// The requests this event's actions fired, as trace record entries.
+// `context.requests[requestId]` is a newest-first call history whose entries
+// carry the block and the id of the action that made the call, so the newest
+// call of a request belongs to this event when isEventCall says so.
+function collectEventRequests({ blockId, context, responses }) {
+  const requests = [];
+  Object.keys(context?.requests ?? {}).forEach((requestId) => {
+    const call = (context.requests[requestId] ?? [])[0];
+    if (!isEventCall({ blockId, call, responses })) return;
+    requests.push({
+      id: requestId,
+      ok: type.isNone(call.error),
+      ms: type.isNumber(call.responseTime) ? call.responseTime : null,
+    });
+  });
+  return requests;
+}
+
+export default collectEventRequests;

package/client/recorder/createPairingBuffer.js

@@ -0,0 +1,187 @@
+/*
+  Copyright 2020-2026 Lowdefy, Inc
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+import { pairTraceEvents, type } from '@lowdefy/helpers';
+
+import buildEngineRecord from './buildEngineRecord.js';
+import buildInteractionRecord from './buildInteractionRecord.js';
+import buildTraceEvent from './buildTraceEvent.js';
+
+// How long an interaction waits for the events it caused. The pairing rule
+// decides by when an event's actions started (within its debounce plus 1 s of
+// the interaction), but the trace hook reports an event when its actions
+// finish, and a click's Request can take seconds. Holding each interaction
+// this long lets a slow event, and a `bubble: true` sibling, still join it.
+const HOLD_MS = 10000;
+const PAIRED_KINDS = ['click', 'change', 'key'];
+
+function hasTargetIdentity(target) {
+  return (
+    !type.isNone(target) &&
+    ((type.isString(target.block_id) && target.block_id !== '') ||
+      (type.isString(target.text) && target.text !== ''))
+  );
+}
+
+// Turns DOM interactions and engine trace payloads into finished v1 trace
+// records, paired in the tab with the shared pairing rule. Each record carries
+// the build getBuild() named when its interaction was captured or its event
+// arrived, never when its hold closes: a hold can outlast a config reload.
+//
+// - addInteraction({ t, kind, element, target, pageId, pathParams, value, key, url, build }):
+//   pageview and back are recorded at once; click, change and key wait until
+//   their hold ends, collecting the events the rule pairs with them. `build`
+//   is given only by a caller that knows it better than getBuild() does now
+//   (a pageview, recorded once its page config has rendered).
+// - addEvent(payload, { urlAfter }): the event is built into a trace event
+//   now (its state writes are read against the state as it is now), then
+//   paired with the latest open interaction that could have caused it, or
+//   recorded at once as kind engine. Every interaction that could cause an
+//   event happened before the event started, so it is already in the buffer.
+// - flushAll(): ends every hold now (pagehide, config reload, headless flush).
+function createPairingBuffer({
+  onRecord,
+  now = Date.now,
+  getSession,
+  getRoles = () => [],
+  getBuild = () => null,
+  redactor,
+}) {
+  let open = [];
+  let nextId = 1;
+  let timer = null;
+
+  function emit(record) {
+    onRecord(redactor === undefined ? record : redactor.redact(record));
+  }
+
+  function finalise(interaction) {
+    let paired = [];
+    if (interaction.events.length > 0) {
+      const { pairs } = pairTraceEvents({
+        interactions: [interaction],
+        events: interaction.events,
+      });
+      const order = pairs[0]?.eventIds ?? interaction.events.map((event) => event.id);
+      paired = order.map((id) => interaction.events.find((event) => event.id === id).traceEvent);
+    }
+    emit(
+      buildInteractionRecord({
+        interaction,
+        paired,
+        session: getSession(),
+        roles: getRoles(),
+      })
+    );
+  }
+
+  function schedule() {
+    if (timer !== null) {
+      clearTimeout(timer);
+      timer = null;
+    }
+    if (open.length === 0) return;
+    const next = Math.min(...open.map((interaction) => interaction.closesAt));
+    timer = setTimeout(closeDue, Math.max(0, next - now()));
+  }
+
+  function closeDue() {
+    timer = null;
+    const time = now();
+    const due = open.filter((interaction) => interaction.closesAt <= time);
+    open = open.filter((interaction) => interaction.closesAt > time);
+    due.forEach(finalise);
+    schedule();
+  }
+
+  function addInteraction({
+    t,
+    kind,
+    element,
+    target,
+    pageId,
+    pathParams,
+    value,
+    key,
+    url,
+    build: named,
+  }) {
+    const time = t ?? now();
+    const build = type.isUndefined(named) ? getBuild() : named;
+    if (!PAIRED_KINDS.includes(kind)) {
+      emit(
+        buildInteractionRecord({
+          interaction: { t: time, kind, pageId, pathParams, url, target: null, build },
+          session: getSession(),
+          roles: getRoles(),
+        })
+      );
+      return;
+    }
+    // A click or change the runner could not target (no block, no text) can
+    // be neither replayed nor paired.
+    if (kind !== 'key' && !hasTargetIdentity(target)) return;
+    if (redactor !== undefined && !type.isNone(element) && redactor.isPassword(element)) {
+      redactor.remember(target?.block_id);
+    }
+    open.push({
+      id: `i${nextId++}`,
+      t: time,
+      kind,
+      pageId,
+      value,
+      key,
+      target: kind === 'key' && !hasTargetIdentity(target) ? null : target,
+      blockIds: target?.block_ids ?? [],
+      build,
+      events: [],
+      closesAt: time + HOLD_MS,
+    });
+    schedule();
+  }
+
+  function addEvent(payload, { urlAfter } = {}) {
+    const engineEvent = {
+      id: `e${nextId++}`,
+      blockId: payload.blockId,
+      eventName: payload.eventName,
+      startTimestamp: new Date(payload.record?.startTimestamp ?? now()).getTime(),
+      debounceMs: payload.debounceMs ?? 0,
+      scope: payload.scope,
+      pageId: payload.pageId,
+      build: getBuild(),
+      traceEvent: buildTraceEvent({ payload, urlAfter }),
+    };
+    const { pairs } = pairTraceEvents({ interactions: open, events: [engineEvent] });
+    if (pairs.length === 0) {
+      emit(buildEngineRecord({ engineEvent, session: getSession(), roles: getRoles() }));
+      return;
+    }
+    open.find((interaction) => interaction.id === pairs[0].interactionId).events.push(engineEvent);
+  }
+
+  function flushAll() {
+    const all = open;
+    open = [];
+    all.forEach(finalise);
+    schedule();
+  }
+
+  return { addEvent, addInteraction, flushAll };
+}
+
+export { HOLD_MS };
+export default createPairingBuffer;