dsh-moli 1.1.12 → 1.1.13

package/index.js

@@ -1,8 +1,6 @@
 /** Host half of dsh-moli: expose the local `moli` headless browser as callable tools. */
-import { performAction } from './lib/actions.js';
 import { MANUAL_INSTALL, installMoli } from './lib/install.js';
 import { findMoli, moliCandidates, probe, runFetch } from './lib/moli.js';
-import { SessionManager, sessionSupported, sessionUnsupportedReason } from './lib/session.js';
 
 export const name = 'dsh-moli';
 export const inject = ['tools'];
@@ -69,6 +67,10 @@ const FETCH_PARAMETERS = {
       type: 'string',
       description: 'JavaScript expression evaluated once the page is ready; its value is returned as text or JSON instead of the page. Combining it with `dump` does NOT return both: the engine discards the expression value and dumps the resulting page state instead (its own documented semantics). Use `eval` alone to read a value, or `dump` alone to read the page.',
     },
+    scroll: {
+      type: 'string',
+      description: 'Scroll the page before capturing it: a number of pixels down as a string ("800", "-500" to go up), or "top" / "bottom". A page that loads content on its first scroll event shows a placeholder without this and the loaded content with it, so use it for lazy lists, infinite scroll, and content revealed by an intersection observer. Runs as part of the engine\'s own pre-capture evaluation, so it happens before any `eval` expression reads the page.',
+    },
     wait: {
       type: 'string',
       enum: ['domcontentloaded', 'load', 'networkidle', 'domstable', 'done'],
@@ -228,174 +230,6 @@ function renderProbe(_args, value) {
   return [{ type: 'text', text: lines.join('\n') }];
 }
 
-/** Actions `moli_session` accepts. */
-const SESSION_ACTIONS = ['start', 'open', 'snapshot', 'click', 'type', 'press', 'scroll', 'waitFor', 'eval', 'cdp', 'screenshot', 'status', 'close'];
-
-/**
- * Parameters of `moli_session`.
- *
- * Raw JSON Schema with a root `type: "object"`, like the other tools: `schemaOf`
- * projects this straight to the provider, so the author DSL would arrive as
- * `type: null` and be rejected.
- */
-const SESSION_PARAMETERS = {
-  type: 'object',
-  additionalProperties: false,
-  required: ['action'],
-  properties: {
-    action: {
-      type: 'string',
-      enum: SESSION_ACTIONS,
-      description: 'start: ensure the browser is running. open: navigate. snapshot: read the rendered page. '
-        + 'click/type/press/scroll: interact. waitFor: wait for a selector to appear or disappear. '
-        + 'eval: run JavaScript in the page. cdp: send a raw Chrome DevTools Protocol command. '
-        + 'screenshot: write a PNG. status: report the session and which browser it uses. close: stop the browser.',
-    },
-    url: { type: 'string', description: 'open: absolute http:// or https:// URL to navigate to.' },
-    selector: { type: 'string', description: 'CSS selector for click, type, scroll and waitFor.' },
-    text: { type: 'string', description: 'type: the text to enter.' },
-    submit: { type: 'boolean', description: 'type: press Enter after entering the text.' },
-    clear: { type: 'boolean', description: 'type: empty the field before entering text.' },
-    key: { type: 'string', description: 'press: key name, e.g. Enter, Tab, Escape, ArrowDown.' },
-    dy: { type: 'number', description: 'scroll: pixels to move down (negative moves up). Defaults to 800.' },
-    block: { type: 'string', enum: ['start', 'center', 'end', 'nearest'], description: 'scroll: where to align the element. Defaults to center.' },
-    state: { type: 'string', enum: ['attached', 'detached'], description: 'waitFor: wait for the selector to appear (attached) or vanish (detached).' },
-    expression: { type: 'string', description: 'eval: a JavaScript expression; promises are awaited and the value is returned.' },
-    method: { type: 'string', description: 'cdp: the protocol method to invoke, e.g. "Network.enable" or "DOM.setFileInputFiles".' },
-    params: { description: 'cdp: the method parameters, as an object or a JSON string. Optional.' },
-    writeTo: { type: 'string', description: 'screenshot: absolute path for the PNG. Required, since images are not returned inline.' },
-    format: { type: 'string', enum: ['png', 'jpeg'], description: 'screenshot: image format. Defaults to png.' },
-    includeHtml: { type: 'boolean', description: 'snapshot: also return the rendered HTML.' },
-    maxChars: { type: 'integer', description: 'snapshot and cdp: truncate text output to this many characters.' },
-    timeoutMs: { type: 'integer', description: 'Deadline for this action. Defaults to the plugin configuration.' },
-    port: { type: 'integer', description: 'start: CDP port. Defaults to the plugin configuration (9222).' },
-    cdpEndpoint: {
-      type: 'string',
-      description: 'Attach to a browser that is already running, e.g. "http://127.0.0.1:9222", instead of starting one. This is how you drive your own Chrome with its logins: start it with --remote-debugging-port and pass that URL.',
-    },
-  },
-};
-
-const SESSION_OUTPUT_SCHEMA = {
-  type: 'object',
-  additionalProperties: true,
-  required: ['ok'],
-  properties: {
-    ok: { type: 'boolean' },
-    action: { type: 'string' },
-    error: { type: 'string' },
-    hint: { type: 'string' },
-    url: { type: 'string' },
-    title: { type: 'string' },
-    text: { type: 'string' },
-    chars: { type: 'integer' },
-    truncated: { type: 'boolean' },
-    links: { type: 'array', items: { type: 'object', additionalProperties: true } },
-    fields: { type: 'array', items: { type: 'object', additionalProperties: true } },
-    html: { type: 'string' },
-    selector: { type: 'string' },
-    clicked: { type: 'object', additionalProperties: true },
-    navigated: { type: 'boolean' },
-    submitted: { type: 'boolean' },
-    value: {},
-    serialized: { type: 'string' },
-    key: { type: 'string' },
-    y: { type: 'number' },
-    height: { type: 'number' },
-    viewport: { type: 'number' },
-    state: { type: 'string' },
-    waitedMs: { type: 'integer' },
-    path: { type: 'string' },
-    bytes: { type: 'integer' },
-    format: { type: 'string' },
-    method: { type: 'string' },
-    result: {},
-    bytes: { type: 'integer' },
-    backend: { type: 'string' },
-    geometry: { type: 'string' },
-    browserVersion: { type: 'string' },
-    running: { type: 'boolean' },
-    ownedByThisPlugin: { type: 'boolean' },
-    external: { type: 'boolean' },
-    pid: { type: 'integer' },
-    endpoint: { type: 'string' },
-    logPath: { type: 'string' },
-    settled: { type: 'boolean' },
-    settledMs: { type: 'integer' },
-    stopped: { type: 'boolean' },
-    scope: { type: 'string' },
-  },
-};
-
-/**
- * Render a `moli_session` value for the model.
- *
- * A snapshot is the one result worth formatting rather than dumping as JSON: the
- * link and field tables are what the next action usually depends on.
- *
- * @param _args - the call arguments, unused.
- * @param value - the canonical result value.
- * @returns result content blocks.
- */
-function renderSession(_args, value) {
-  if (value.ok !== true) {
-    const lines = [`moli_session failed${typeof value.action === 'string' ? ` (${value.action})` : ''}: ${value.error ?? 'unknown error'}`];
-    if (typeof value.hint === 'string' && value.hint !== '') lines.push(value.hint);
-    if (typeof value.bytes === 'number') lines.push(`(the image was ${String(value.bytes)} bytes but could not be written)`);
-    return [{ type: 'text', text: lines.join('\n') }];
-  }
-  if (value.action === 'snapshot') {
-    const lines = [`${String(value.title)} — ${String(value.url)}`, ''];
-    lines.push(String(value.text ?? ''));
-    if (Array.isArray(value.links) && value.links.length > 0) {
-      lines.push('', `Links (${String(value.links.length)}):`);
-      for (const link of value.links.slice(0, 25)) lines.push(`  ${String(link.text)} -> ${String(link.href)}`);
-    }
-    if (Array.isArray(value.fields) && value.fields.length > 0) {
-      lines.push('', `Fields (${String(value.fields.length)}):`);
-      for (const field of value.fields.slice(0, 25)) lines.push(`  <${String(field.tag)}${field.type === '' ? '' : ` type=${String(field.type)}`}${field.name === '' ? '' : ` name=${String(field.name)}`}${field.id === '' ? '' : ` id=${String(field.id)}`}>`);
-    }
-    return [{ type: 'text', text: lines.join('\n') }];
-  }
-  if (value.action === 'status') {
-    if (value.running !== true) {
-      return [{ type: 'text', text: 'No session is running. Call moli_session with action "start" or "open" to launch one.' }];
-    }
-    const lines = [
-      `Session running: ${String(value.backend)} (${String(value.browserVersion ?? 'version unknown')}) at ${String(value.endpoint)}`,
-    ];
-    if (value.geometry === 'absolute') {
-      lines.push('Geometry: moli\'s document-absolute frame — element input cannot reach below the first viewport. Use `browser: "chromium"` to lift that, or eval to click in the page.');
-    }
-    if (value.external === true) lines.push('Started outside this plugin, so close will not stop it.');
-    if (value.url !== undefined) lines.push(`Current page: ${String(value.title)} — ${String(value.url)}`);
-    return [{ type: 'text', text: lines.join('\n') }];
-  }
-  if (value.action === 'cdp') {
-    return [{ type: 'text', text: `${String(value.method)} → ${String(value.serialized ?? '').slice(0, 4000)}` }];
-  }
-  if (value.action === 'click') {
-    return [{
-      type: 'text',
-      text: `Clicked ${String(value.clicked?.tag ?? 'element')}${value.clicked?.text === undefined || value.clicked.text === '' ? '' : ` (“${String(value.clicked.text)}”)`}`
-        + `${value.navigated === true ? `, which navigated to ${String(value.url)}` : '; the page did not navigate'}`,
-    }];
-  }
-  if (value.action === 'type') {
-    return [{ type: 'text', text: `Entered text${value.submitted === true ? ' and pressed Enter' : ''}; the field now reads ${JSON.stringify(String(value.value ?? ''))}` }];
-  }
-  if (value.action === 'screenshot') {
-    return [{ type: 'text', text: `Screenshot written to ${String(value.path)} (${String(value.bytes)} bytes). Read that file to see it.` }];
-  }
-  if (value.action === 'eval') {
-    return [{ type: 'text', text: String(value.serialized ?? '') }];
-  }
-  if (value.action === 'close') {
-    return [{ type: 'text', text: value.stopped === true ? 'Session stopped.' : 'Nothing to stop: no session was running that this plugin started.' }];
-  }
-  return [{ type: 'text', text: JSON.stringify(value) }];
-}
-
 /**
  * Register the Moli tools.
  *
@@ -525,133 +359,5 @@ export function apply(ctx, config) {
     },
   });
 
-  // ---------------------------------------------------------------------------
-  // moli_session: an interactive browser, for the work a one-shot fetch cannot do
-  // ---------------------------------------------------------------------------
-  //
-  // This is the plugin's only stateful tool. `moli serve` is a long-lived child
-  // that has to outlive a single call, so one manager is created per plugin
-  // instance and its lifecycle is explicit (`start` implicitly, `close` by hand).
-  // Registration stays synchronous: the server is spawned on first use, never at
-  // load time, so a plugin load cannot delay Host startup or open a port.
-  const session = new SessionManager({
-    get exe() { return cachedExe; },
-    // Which browser drives a session. Explicit, never automatic: switching engines
-    // because one click failed would make the timing and the behaviour depend on
-    // which operation happened to fail, and the caller could not tell which browser
-    // it was talking to.
-    browser: options.sessionBrowser ?? 'moli',
-    browserPath: options.sessionBrowserPath,
-    cdpEndpoint: options.sessionCdpEndpoint,
-    port: options.sessionPort ?? 9222,
-    host: '127.0.0.1',
-    insecureTls,
-    proxy: options.proxy,
-    userAgent: options.sessionUserAgent,
-    profileDir: options.profileDir,
-    resources: options.sessionResources === true,
-    // Off, matching the engine's own default: `moli fetch <private-ip>` can reach
-    // a private address, so the plugin does not silently narrow what a user's
-    // installed engine already does. Set it to true, or pass blockCidrs, to have
-    // the engine refuse private addresses for directly connected targets.
-    blockPrivateNetworks: options.blockPrivateNetworks === true,
-    blockCidrs: Array.isArray(options.blockCidrs) ? options.blockCidrs : [],
-    startTimeoutMs: options.sessionStartTimeoutMs ?? 20000,
-  });
-
-  ctx.tools.register({
-    name: 'moli_session',
-    description: 'Drive an interactive browser through a persistent session, for what a one-shot fetch cannot'
-      + ' do: click through consent walls, fill and submit a search box, scroll a lazy list, wait for a'
-      + ' client-side route change, or run a script in the page. Keeps one browser alive across calls, so'
-      + ' state (cookies, navigation, form input) persists until you stop it with action "close".'
-      + ' Start with "open"; read the page with "snapshot"; interact with "click", "type", "press" and'
-      + ' "scroll"; use "eval" as the escape hatch. Elements are addressed by CSS selector, never by pixel'
-      + ' coordinates.',
-    parameters: SESSION_PARAMETERS,
-    output: { schema: SESSION_OUTPUT_SCHEMA, render: renderSession },
-    // One browser, one page: overlapping actions would interleave input events and
-    // navigate the page out from under each other.
-    isConcurrencySafe: () => false,
-    async execute(args, exec) {
-      // A per-call endpoint is honoured by every action, including the ones that
-      // never reach `performAction`. Routing `close` through it is not cosmetic:
-      // `withEndpoint` is what marks the connection external, and `close` refuses
-      // to stop an external browser. Answering `close` before that mark existed
-      // meant a call that had just attached to the user's own Chrome could shut it
-      // down -- the one thing this design promises not to do.
-      const withCallEndpoint = work => (typeof args.cdpEndpoint === 'string' && args.cdpEndpoint !== ''
-        ? session.withEndpoint(args.cdpEndpoint, work)
-        : work());
-
-      // `status` and `close` are meaningful with nothing running, so they are
-      // answered before the engine is required.
-      if (args.action === 'close') {
-        return withCallEndpoint(async () => ({ ok: true, action: 'close', ...(await session.close()) }));
-      }
-      if (args.action === 'status') {
-        return withCallEndpoint(async () => ({ ok: true, action: 'status', ...(await session.status()) }));
-      }
-      if (args.action === 'start') {
-        if (!sessionSupported()) return { ok: false, action: 'start', error: sessionUnsupportedReason() };
-        return withCallEndpoint(async () => {
-          const { external, backend, geometry } = await session.ensure();
-          const state = await session.status();
-          return { ok: true, action: 'start', started: true, external, backend, geometry, ...state };
-        });
-      }
-
-      if (engine() === undefined && (options.sessionBrowser ?? 'moli') === 'moli'
-        && (options.sessionCdpEndpoint === undefined || options.sessionCdpEndpoint === '')) {
-        return {
-          ok: false,
-          action: String(args.action),
-          error: 'moli was not found, so a moli session cannot be started',
-          hint: 'Call moli_probe with action "install", set the plugin config field `exePath`, or set `sessionBrowser: "chromium"` to drive the browser you already have.',
-        };
-      }
-      if (!sessionSupported()) return { ok: false, action: String(args.action), error: sessionUnsupportedReason() };
-
-      if (exec?.signal?.aborted === true) return { ok: false, action: String(args.action), error: 'the call was aborted before it started' };
-
-      try {
-        const perform = async ({ connection, geometry, backend }) => {
-          try {
-            return await performAction(connection, args, { maxChars, geometry, backend });
-          } finally {
-            // A closed socket means the browser went away under us; drop it so the
-            // next call starts a fresh one instead of failing forever.
-            if (connection.closed === true) void session.close();
-          }
-        };
-        // A per-call endpoint drives a browser the caller already has open, which is
-        // the only way to inherit their logins. It is scoped to this call: an
-        // endpoint that latched would make one bad call poison every later one.
-        if (typeof args.cdpEndpoint === 'string' && args.cdpEndpoint !== '') {
-          return await session.withEndpoint(args.cdpEndpoint, perform);
-        }
-        return await perform(await session.ensure());
-      } catch (error) {
-        const message = error instanceof Error ? error.message : String(error);
-        const hint = /timed out/u.test(message)
-          ? 'The action exceeded its deadline. Raise timeoutMs, or check `waitFor` for a selector that never appears.'
-          : /no Chromium-family browser/u.test(message)
-            ? 'Install Chrome, Edge or Brave, or set the plugin config field `sessionBrowserPath` to a browser executable.'
-            : /ECONNREFUSED|closed|WebSocket/u.test(message)
-              ? 'The browser connection was lost. Call moli_session with action "status", then "start" to relaunch it.'
-              : undefined;
-        return { ok: false, action: String(args.action), error: message, ...(hint === undefined ? {} : { hint }) };
-      }
-    },
-  });
-
-  // A serve process or a headless browser outlives a tool call, so it must be tied
-  // to the plugin's lifetime rather than left to the OS. Cordis disposes listeners
-  // registered through `ctx.on` together with the plugin's fiber, so registering
-  // one is enough; the returned disposer is not needed here.
-  if (typeof ctx.on === 'function') {
-    ctx.on('dispose', () => { void session.close(); });
-  }
-
-  ctx.logger?.info?.(`dsh-moli: registered moli_fetch, moli_probe and moli_session (${cachedExe ?? 'moli missing'})`);
+  ctx.logger?.info?.(`dsh-moli: registered moli_fetch and moli_probe (${cachedExe ?? 'moli missing'})`);
 }

package/lib/moli.js

@@ -31,6 +31,64 @@ const TEXT_DUMPS = new Set([undefined, 'html', 'markdown', 'json', 'semantic_tre
 /** Accepted `--wait` lifecycle stages. */
 const WAIT_STAGES = new Set(['domcontentloaded', 'load', 'networkidle', 'domstable', 'done']);
 
+/** `scroll` keywords, mapped to what they mean in the page. */
+const SCROLL_TOKENS = {
+  top: '0',
+  bottom: 'document.documentElement.scrollHeight',
+};
+
+/**
+ * Build the expression that scrolls the page, to be run by the engine's own
+ * `--eval` before the dump.
+ *
+ * Scrolling is not a nicety: a page can load content on its first scroll event, and
+ * then a fetch without scrolling sees a placeholder while the same fetch with it
+ * sees the content. Verified against a page whose scroll handler fills a slot --
+ * the dump carried the loaded text with `scroll` and the placeholder without it.
+ * Moli does serialize the whole document either way, so this is not about reaching
+ * content that "below the fold" already contains; it is about triggering the
+ * handler, the intersection observer, or the lazy image that only runs on a scroll.
+ *
+ * @param scroll - pixels down (negative scrolls up), or `top` / `bottom`.
+ * @returns a JavaScript expression, or undefined when there is nothing to do.
+ */
+export function scrollScript(scroll) {
+  if (scroll === undefined || scroll === null) return undefined;
+  let target;
+  if (typeof scroll === 'number') {
+    if (!Number.isFinite(scroll)) return undefined;
+    target = String(Math.trunc(scroll));
+  } else if (typeof scroll === 'string') {
+    const key = scroll.trim().toLowerCase();
+    if (key === '') return undefined;
+    target = SCROLL_TOKENS[key] ?? (/^-?\d+$/u.test(key) ? key : undefined);
+    if (target === undefined) return undefined;
+  } else {
+    return undefined;
+  }
+  // Dispatch a scroll event as well as moving: `window.scrollTo` updates the
+  // position, but a page listening for the event is the reason this exists.
+  return `(() => { window.scrollTo(0, ${target}); window.dispatchEvent(new Event('scroll')); return String(window.scrollY); })()`;
+}
+
+/**
+ * Combine the caller's `eval` with a scroll, if one was asked for.
+ *
+ * The two have to share one expression because the engine accepts a single
+ * `--eval`. The scroll goes first so the page is in its final state before the
+ * caller's expression reads it, and the caller's value still becomes the result.
+ *
+ * @returns the expression to pass, or undefined when neither was given.
+ */
+export function combineEval(evalExpression, scroll) {
+  const move = scrollScript(scroll);
+  if (move === undefined) return evalExpression;
+  if (evalExpression === undefined || evalExpression === '') return move;
+  // The caller's expression is kept last and wrapped so its value is the one the
+  // engine reports. Losing it would contradict the documented `eval` behaviour.
+  return `${move}; (() => (${evalExpression}))()`;
+}
+
 /**
  * Every place this plugin will look for `moli.exe`, most explicit first.
  *
@@ -256,7 +314,10 @@ function buildArgs(args) {
   // Screenshots and PDFs silently produce nothing without the layout renderer.
   if (needsLayout) argv.push('--layout');
   if (args.scrollbars === true) argv.push('--scrollbars');
-  if (args.eval !== undefined) argv.push('--eval', args.eval);
+  // `scroll` rides along in the same expression: the engine takes a single `--eval`,
+  // and the scroll has to happen before the page is captured.
+  const evaluation = combineEval(args.eval, args.scroll);
+  if (evaluation !== undefined) argv.push('--eval', evaluation);
   if (args.wait !== undefined && args.wait !== '') argv.push('--wait', args.wait);
   if (args.waitSelector !== undefined && args.waitSelector !== '') argv.push('--wait-selector', args.waitSelector);
   if (args.delayMs !== undefined) argv.push('--delay-ms', String(args.delayMs));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dsh-moli",
-  "version": "1.1.12",
+  "version": "1.1.13",
   "type": "module",
   "description": "DeepSeek Harness tools for the local Moli headless browser: rendered fetches, markdown, semantic trees, screenshots, and PDFs.",
   "license": "MIT",