@lowdefy/server-dev 5.6.0 → 6.0.0

package/lib/docs/checkpointStore.js

@@ -0,0 +1,240 @@
+/*
+  Copyright 2020-2026 Lowdefy, Inc
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+import fs from 'fs';
+import path from 'path';
+import { isReserved, serializer, setKey, type } from '@lowdefy/helpers';
+
+import { getConfigDirectory } from './checkpointPaths.js';
+
+// State checkpoints (page state + recorded request/api responses) live under
+// <configDir>/.lowdefy/state-checkpoints/<name>/ — inside .lowdefy because a
+// checkpoint captures the signed-in user's data and live backend responses,
+// which must not sit in a committable spot by default (.lowdefy is gitignored
+// by convention and ignored by the dev file watcher). A sibling of
+// .lowdefy/checkpoints (the config-directory-snapshot checkpoints in
+// checkpointPaths.js/createConfigCheckpoint.js) rather than a subdirectory,
+// because that store prunes its oldest directories beyond a cap. These two
+// "checkpoint" concepts are unrelated: this one is for putting the *running
+// app* into a specific state (agent verification, human testing, e2e test
+// generation), not for reverting config files.
+const NAME_PATTERN = /^[a-z0-9-_]+$/i;
+
+function getStateCheckpointsRoot() {
+  return path.join(getConfigDirectory(), '.lowdefy', 'state-checkpoints');
+}
+
+function validateName(name) {
+  if (type.isNone(name) || !type.isString(name)) {
+    throw new Error(`Checkpoint "name" must be a string. Received ${JSON.stringify(name)}.`);
+  }
+  if (!NAME_PATTERN.test(name)) {
+    throw new Error(
+      `Checkpoint name "${name}" is invalid — use only letters, numbers, "-" and "_".`
+    );
+  }
+}
+
+function writeJsonFile(filePath, value) {
+  fs.mkdirSync(path.dirname(filePath), { recursive: true });
+  fs.writeFileSync(filePath, JSON.stringify(serializer.serialize(value), null, 2));
+}
+
+function readJsonFile(filePath, defaultValue) {
+  if (!fs.existsSync(filePath)) {
+    return defaultValue;
+  }
+  return serializer.deserialize(JSON.parse(fs.readFileSync(filePath, 'utf8')));
+}
+
+// Requests/api responses arrive as either an array of entries (newest first —
+// the shape of context.requests in both Requests.js and Inspector.jsx's
+// buildSnapshot) or a single entry object. Either way only the newest
+// recording is kept per id.
+function newestEntry(value) {
+  return Array.isArray(value) ? value[0] : value;
+}
+
+function writeEntries({ dir, subdir, entries, parts }) {
+  Object.entries(entries ?? {}).forEach(([id, value]) => {
+    const entry = newestEntry(value);
+    if (type.isNone(entry)) {
+      return;
+    }
+    const { payload, response, error, responseTime } = entry;
+    writeJsonFile(path.join(dir, subdir, `${id}.json`), { payload, response, error, responseTime });
+    parts.push(`${subdir}/${id}.json`);
+  });
+}
+
+function readEntries({ dir, subdir }) {
+  const entriesDir = path.join(dir, subdir);
+  const entries = {};
+  if (!fs.existsSync(entriesDir)) {
+    return entries;
+  }
+  fs.readdirSync(entriesDir)
+    .filter((fileName) => fileName.endsWith('.json'))
+    .forEach((fileName) => {
+      const id = fileName.slice(0, -'.json'.length);
+      // A checkpoint directory is on-disk state a developer can edit by hand.
+      // A reserved file name can't round-trip through the entries map, so skip
+      // it rather than fail the whole checkpoint load.
+      if (isReserved(id)) {
+        return;
+      }
+      setKey(entries, id, readJsonFile(path.join(entriesDir, fileName), null));
+    });
+  return entries;
+}
+
+// Snapshots a running page's state + recorded request/api responses into a
+// checkpoint folder. Refuses to clobber an existing checkpoint unless
+// `overwrite: true` — when overwriting, the old folder is removed first so a
+// checkpoint written with fewer requests than a previous one doesn't leave
+// stale per-request files behind.
+function writeCheckpoint({ name, snapshot, notes, overwrite }) {
+  validateName(name);
+  if (type.isNone(snapshot) || !type.isObject(snapshot)) {
+    throw new Error(
+      `writeCheckpoint requires a "snapshot" object. Received ${JSON.stringify(snapshot)}.`
+    );
+  }
+
+  const dir = path.join(getStateCheckpointsRoot(), name);
+  const exists = fs.existsSync(dir);
+  if (exists && overwrite !== true) {
+    throw new Error(`Checkpoint "${name}" already exists. Pass overwrite: true to replace it.`);
+  }
+  if (exists) {
+    fs.rmSync(dir, { recursive: true, force: true });
+  }
+  fs.mkdirSync(dir, { recursive: true });
+
+  const parts = [];
+  const manifest = {
+    name,
+    pageId: snapshot.pageId ?? null,
+    url: snapshot.url ?? null,
+    capturedAt: new Date().toISOString(),
+    notes: notes ?? null,
+  };
+  writeJsonFile(path.join(dir, 'checkpoint.json'), manifest);
+  parts.push('checkpoint.json');
+
+  writeJsonFile(path.join(dir, 'state.json'), snapshot.state ?? {});
+  parts.push('state.json');
+  writeJsonFile(path.join(dir, 'urlQuery.json'), snapshot.urlQuery ?? '');
+  parts.push('urlQuery.json');
+  writeJsonFile(path.join(dir, 'inputs.json'), snapshot.input ?? {});
+  parts.push('inputs.json');
+  writeJsonFile(path.join(dir, 'user.json'), snapshot.user ?? null);
+  parts.push('user.json');
+  writeJsonFile(path.join(dir, 'global.json'), snapshot.global ?? {});
+  parts.push('global.json');
+
+  writeEntries({ dir, subdir: 'requests', entries: snapshot.requests, parts });
+  if (!type.isNone(snapshot.apiResponses)) {
+    writeEntries({ dir, subdir: 'api', entries: snapshot.apiResponses, parts });
+  }
+
+  return { dir, parts };
+}
+
+// Reads a checkpoint back, tolerant of individually missing parts (a
+// developer may hand-edit or delete files in a committed checkpoint folder) —
+// only a missing checkpoint directory itself is an error.
+function readCheckpoint({ name }) {
+  validateName(name);
+  const dir = path.join(getStateCheckpointsRoot(), name);
+  if (!fs.existsSync(dir)) {
+    throw new Error(`Checkpoint "${name}" not found.`);
+  }
+
+  const checkpoint = readJsonFile(path.join(dir, 'checkpoint.json'), {
+    name,
+    pageId: null,
+    url: null,
+    capturedAt: null,
+    notes: null,
+  });
+
+  return {
+    name,
+    checkpoint,
+    state: readJsonFile(path.join(dir, 'state.json'), {}),
+    urlQuery: readJsonFile(path.join(dir, 'urlQuery.json'), ''),
+    input: readJsonFile(path.join(dir, 'inputs.json'), {}),
+    user: readJsonFile(path.join(dir, 'user.json'), null),
+    global: readJsonFile(path.join(dir, 'global.json'), {}),
+    requests: readEntries({ dir, subdir: 'requests' }),
+    api: readEntries({ dir, subdir: 'api' }),
+  };
+}
+
+function checkpointExists({ name }) {
+  validateName(name);
+  return fs.existsSync(path.join(getStateCheckpointsRoot(), name));
+}
+
+// Lists stored state checkpoints, most recent first. Corrupted/hand-edited
+// checkpoints without a manifest still show up (with null fields) rather
+// than being silently hidden, since these folders are meant to be
+// human-editable.
+function listStateCheckpoints() {
+  const root = getStateCheckpointsRoot();
+  if (!fs.existsSync(root)) {
+    return [];
+  }
+
+  const checkpoints = fs
+    .readdirSync(root, { withFileTypes: true })
+    .filter((entry) => entry.isDirectory())
+    .map((entry) => {
+      const dir = path.join(root, entry.name);
+      const manifest = readJsonFile(path.join(dir, 'checkpoint.json'), {
+        name: entry.name,
+        pageId: null,
+        capturedAt: null,
+        notes: null,
+      });
+      const requestsDir = path.join(dir, 'requests');
+      const requestCount = fs.existsSync(requestsDir)
+        ? fs.readdirSync(requestsDir).filter((fileName) => fileName.endsWith('.json')).length
+        : 0;
+      return {
+        name: entry.name,
+        pageId: manifest.pageId ?? null,
+        capturedAt: manifest.capturedAt ?? null,
+        notes: manifest.notes ?? null,
+        requestCount,
+      };
+    });
+
+  return checkpoints.sort((a, b) => {
+    if (type.isNone(a.capturedAt)) return 1;
+    if (type.isNone(b.capturedAt)) return -1;
+    return a.capturedAt < b.capturedAt ? 1 : -1;
+  });
+}
+
+export {
+  checkpointExists,
+  getStateCheckpointsRoot,
+  listStateCheckpoints,
+  readCheckpoint,
+  writeCheckpoint,
+};

package/lib/docs/checkpointToMocks.js

@@ -0,0 +1,65 @@
+/*
+  Copyright 2020-2026 Lowdefy, Inc
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+import YAML from 'yaml';
+
+import { readCheckpoint } from './checkpointStore.js';
+
+// Errors can't round-trip through YAML as Error instances (and don't need
+// to) — @lowdefy/e2e-utils' createMockManager accepts a plain string and
+// wraps it itself (`error instanceof Error ? error : new Error(error)`), so
+// only the message is kept here.
+function toMockError(error) {
+  if (!error) {
+    return undefined;
+  }
+  return error.message ?? String(error);
+}
+
+function toMockEntries(entries, idKey, extraFields = {}) {
+  return Object.entries(entries ?? {}).map(([id, entry]) => {
+    const error = toMockError(entry?.error);
+    const base = { [idKey]: id, ...extraFields };
+    if (error) {
+      return { ...base, error };
+    }
+    return { ...base, response: entry?.response ?? null };
+  });
+}
+
+// Converts a state checkpoint's recorded requests/api responses into the
+// exact shape @lowdefy/e2e-utils' loadStaticMocks/createMockManager expect
+// (`{ requests: [...], api: [...] }`), so a checkpoint recorded from a live
+// dev session can seed a future AI-generated (or hand-written) e2e test's
+// mocks file directly.
+function checkpointToMocks({ name }) {
+  let checkpoint;
+  try {
+    checkpoint = readCheckpoint({ name });
+  } catch (error) {
+    return { error: error.message };
+  }
+
+  const pageId = checkpoint.checkpoint.pageId;
+  const requests = toMockEntries(checkpoint.requests, 'requestId', { pageId });
+  const api = toMockEntries(checkpoint.api, 'endpointId');
+
+  const yamlContent = YAML.stringify({ requests, api });
+
+  return { requests, api, yaml: yamlContent };
+}
+
+export default checkpointToMocks;

package/lib/docs/clientErrorStore.js

@@ -0,0 +1,36 @@
+/*
+  Copyright 2020-2026 Lowdefy, Inc
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+// Module-level ring buffer of recent client-reported errors — feeds the
+// getBuildStatus feedback endpoint so agents can see browser errors without
+// tailing server logs. Deliberately in-memory only: entries are lost on
+// server restart, which is fine since this is a live-session debugging aid.
+const MAX_ENTRIES = 50;
+
+const entries = [];
+
+function push(entry) {
+  entries.push(entry);
+  if (entries.length > MAX_ENTRIES) {
+    entries.shift();
+  }
+}
+
+function list() {
+  return [...entries];
+}
+
+export default { push, list };

package/lib/docs/configCheckpoints.test.mjs

@@ -0,0 +1,156 @@
+/*
+  Copyright 2020-2026 Lowdefy, Inc
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+import fs from 'fs';
+import os from 'os';
+import path from 'path';
+
+import { MAX_CHECKPOINTS } from './checkpointPaths.js';
+import createConfigCheckpoint from './createConfigCheckpoint.js';
+import listConfigCheckpoints from './listConfigCheckpoints.js';
+import revertConfigCheckpoint from './revertConfigCheckpoint.js';
+
+let configDirectory;
+let previousConfigDirectory;
+let previousCwd;
+
+function writeFile(relativePath, content) {
+  const filePath = path.join(configDirectory, relativePath);
+  fs.mkdirSync(path.dirname(filePath), { recursive: true });
+  fs.writeFileSync(filePath, content);
+}
+
+function readFile(relativePath) {
+  return fs.readFileSync(path.join(configDirectory, relativePath), 'utf8');
+}
+
+function exists(relativePath) {
+  return fs.existsSync(path.join(configDirectory, relativePath));
+}
+
+// Checkpoint ids are timestamp-prefixed (see createConfigCheckpoint.js), so
+// tests asserting ordering/eviction need genuinely distinct timestamps
+// between calls. A real (short) delay is used instead of mocking the clock —
+// this Node/Jest combination's fake timers cannot redefine `performance`.
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+beforeEach(() => {
+  configDirectory = fs.mkdtempSync(path.join(os.tmpdir(), 'lowdefy-config-checkpoints-test-'));
+  previousConfigDirectory = process.env.LOWDEFY_DIRECTORY_CONFIG;
+  process.env.LOWDEFY_DIRECTORY_CONFIG = configDirectory;
+  previousCwd = process.cwd();
+  process.chdir(configDirectory);
+
+  writeFile('lowdefy.yaml', 'name: my-app\n');
+  writeFile(path.join('pages', 'home.yaml'), 'id: home\n');
+  writeFile('.env', 'SECRET=abc\n');
+  writeFile('README.md', '# My App\n');
+  // Should never be snapshotted.
+  writeFile(path.join('node_modules', 'some-dep', 'index.js'), 'module.exports = {};\n');
+  writeFile(path.join('.git', 'HEAD'), 'ref: refs/heads/main\n');
+});
+
+afterEach(() => {
+  process.chdir(previousCwd);
+  fs.rmSync(configDirectory, { recursive: true, force: true });
+  if (previousConfigDirectory === undefined) {
+    delete process.env.LOWDEFY_DIRECTORY_CONFIG;
+  } else {
+    process.env.LOWDEFY_DIRECTORY_CONFIG = previousConfigDirectory;
+  }
+});
+
+test('createConfigCheckpoint snapshots config-relevant files and excludes build/vcs/dependency directories', () => {
+  const result = createConfigCheckpoint({ label: 'before change' });
+
+  expect(result.fileCount).toBe(4);
+  expect(result.id).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}-\d{3}Z-before-change$/);
+
+  const snapshotDirectory = path.join(configDirectory, '.lowdefy', 'checkpoints', result.id);
+  const manifest = JSON.parse(
+    fs.readFileSync(path.join(snapshotDirectory, 'manifest.json'), 'utf8')
+  );
+  expect(manifest.files.sort()).toEqual(
+    ['lowdefy.yaml', path.join('pages', 'home.yaml'), '.env', 'README.md'].sort()
+  );
+
+  expect(fs.readFileSync(path.join(snapshotDirectory, 'lowdefy.yaml'), 'utf8')).toBe(
+    'name: my-app\n'
+  );
+  expect(fs.existsSync(path.join(snapshotDirectory, 'node_modules'))).toBe(false);
+  expect(fs.existsSync(path.join(snapshotDirectory, '.git'))).toBe(false);
+});
+
+test('listConfigCheckpoints returns an empty array when no checkpoints exist', () => {
+  expect(listConfigCheckpoints()).toEqual([]);
+});
+
+test('listConfigCheckpoints lists checkpoints most recent first', async () => {
+  const first = createConfigCheckpoint({ label: 'first' });
+  await sleep(5);
+  const second = createConfigCheckpoint({ label: 'second' });
+
+  const checkpoints = listConfigCheckpoints();
+  expect(checkpoints).toEqual([
+    { id: second.id, label: 'second', createdAt: expect.any(String), fileCount: 4 },
+    { id: first.id, label: 'first', createdAt: expect.any(String), fileCount: 4 },
+  ]);
+});
+
+test('revertConfigCheckpoint restores modified and deleted files, and deletes files added since the checkpoint', () => {
+  const { id } = createConfigCheckpoint({ label: 'snapshot' });
+
+  // Modify an existing file.
+  writeFile('lowdefy.yaml', 'name: modified-app\n');
+  // Delete a file that was in the checkpoint.
+  fs.rmSync(path.join(configDirectory, 'pages', 'home.yaml'));
+  // Add a new config-relevant file that was not in the checkpoint.
+  writeFile(path.join('pages', 'extra.yaml'), 'id: extra\n');
+
+  const result = revertConfigCheckpoint({ id });
+
+  expect(readFile('lowdefy.yaml')).toBe('name: my-app\n');
+  expect(exists(path.join('pages', 'home.yaml'))).toBe(true);
+  expect(readFile(path.join('pages', 'home.yaml'))).toBe('id: home\n');
+  expect(exists(path.join('pages', 'extra.yaml'))).toBe(false);
+
+  expect(result.restored.sort()).toEqual(
+    ['lowdefy.yaml', path.join('pages', 'home.yaml'), '.env', 'README.md'].sort()
+  );
+  expect(result.deleted).toEqual([path.join('pages', 'extra.yaml')]);
+});
+
+test('revertConfigCheckpoint throws an actionable error when the checkpoint id does not exist', () => {
+  expect(() => revertConfigCheckpoint({ id: 'does-not-exist' })).toThrow(
+    'Config checkpoint "does-not-exist" not found.'
+  );
+});
+
+test('createConfigCheckpoint evicts the oldest checkpoint once more than the cap are stored', async () => {
+  for (let i = 0; i < MAX_CHECKPOINTS + 1; i += 1) {
+    createConfigCheckpoint({ label: `checkpoint-${i}` });
+    await sleep(2);
+  }
+
+  const checkpoints = listConfigCheckpoints();
+  expect(checkpoints).toHaveLength(MAX_CHECKPOINTS);
+  expect(checkpoints.some((checkpoint) => checkpoint.label === 'checkpoint-0')).toBe(false);
+  expect(
+    checkpoints.some((checkpoint) => checkpoint.label === `checkpoint-${MAX_CHECKPOINTS}`)
+  ).toBe(true);
+});

package/lib/docs/createConfigCheckpoint.js

@@ -0,0 +1,92 @@
+/*
+  Copyright 2020-2026 Lowdefy, Inc
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+import fs from 'fs';
+import path from 'path';
+
+import {
+  getCheckpointsRoot,
+  getConfigDirectory,
+  listConfigFiles,
+  MANIFEST_FILE_NAME,
+  MAX_CHECKPOINTS,
+  slug,
+} from './checkpointPaths.js';
+
+// Checkpoint ids sort chronologically as plain strings (ISO timestamp
+// prefix), which is what listConfigCheckpoints and the checkpoint cap rely
+// on — ":" and "." are replaced since they're awkward/unsafe in directory
+// names on some filesystems.
+function buildCheckpointId(label) {
+  const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+  return `${timestamp}-${slug(label)}`;
+}
+
+function copyIntoSnapshot({ configDirectory, snapshotDirectory, relativeFilePath }) {
+  const source = path.join(configDirectory, relativeFilePath);
+  const destination = path.join(snapshotDirectory, relativeFilePath);
+  fs.mkdirSync(path.dirname(destination), { recursive: true });
+  fs.copyFileSync(source, destination);
+}
+
+// Deletes the oldest checkpoints once the stored count exceeds MAX_CHECKPOINTS.
+// Checkpoint directory names sort chronologically, so a plain string sort
+// puts the oldest first.
+function enforceCheckpointCap({ checkpointsRoot }) {
+  const ids = fs
+    .readdirSync(checkpointsRoot, { withFileTypes: true })
+    .filter((entry) => entry.isDirectory())
+    .map((entry) => entry.name)
+    .sort();
+
+  const excessCount = ids.length - MAX_CHECKPOINTS;
+  if (excessCount <= 0) {
+    return;
+  }
+  ids.slice(0, excessCount).forEach((id) => {
+    fs.rmSync(path.join(checkpointsRoot, id), { recursive: true, force: true });
+  });
+}
+
+// Snapshots every config-relevant file in the config directory into
+// <configDir>/.lowdefy/checkpoints/<id>/, alongside a manifest.json listing
+// the files it holds — revertConfigCheckpoint uses that manifest to restore
+// exactly, including deleting files that didn't exist at checkpoint time.
+function createConfigCheckpoint({ label }) {
+  const configDirectory = getConfigDirectory();
+  const checkpointsRoot = getCheckpointsRoot();
+  const id = buildCheckpointId(label);
+  const snapshotDirectory = path.join(checkpointsRoot, id);
+
+  const files = listConfigFiles({ configDirectory });
+
+  fs.mkdirSync(snapshotDirectory, { recursive: true });
+  files.forEach((relativeFilePath) => {
+    copyIntoSnapshot({ configDirectory, snapshotDirectory, relativeFilePath });
+  });
+
+  const createdAt = new Date().toISOString();
+  fs.writeFileSync(
+    path.join(snapshotDirectory, MANIFEST_FILE_NAME),
+    JSON.stringify({ id, label: label ?? null, createdAt, files }, null, 2)
+  );
+
+  enforceCheckpointCap({ checkpointsRoot });
+
+  return { id, fileCount: files.length };
+}
+
+export default createConfigCheckpoint;