@alumbwe/anvil 1.0.20 → 1.0.22

package/vendor/@anvil/common/src/__tests__/foreign-client-signals.test.ts

@@ -0,0 +1,379 @@
+import { describe, expect, test } from 'bun:test'
+
+import {
+  detectForeignAnvilClient,
+  ANVIL_DOWNGRADE_MODEL_ID,
+  ANVIL_SIGNATURE_TOOL_NAMES,
+  GENERIC_TOOL_NAMES,
+  resolveForeignClientDowngrade,
+} from '../constants/foreign-client-signals'
+import { toolNames } from '../tools/constants'
+
+function tools(...names: string[]) {
+  return names.map((name) => ({ type: 'function', function: { name } }))
+}
+
+/** Toolsets observed on real anvil traffic over 24h of DeepSeek V4 Flash. */
+const ANVIL_TOOLSETS = [
+  // CLI / desktop root agent
+  tools(
+    'ask_user',
+    'basher',
+    'browser_use',
+    'code_reviewer_deepseek_flash',
+    'code_searcher',
+    'context_pruner',
+    'file_picker',
+    'glob',
+    'gravity_index',
+    'list_directory',
+    'read_files',
+    'read_subtree',
+  ),
+  // desktop thread agent
+  tools(
+    'basher',
+    'browser_check',
+    'code_reviewer_deepseek_flash',
+    'code_searcher',
+    'context_pruner',
+    'end_turn',
+    'file_picker',
+    'glob',
+    'list_directory',
+    'preview_click',
+    'preview_evaluate',
+  ),
+  // chat surface
+  tools(
+    'context_pruner',
+    'gravity_index',
+    'render_ui',
+    'researcher_web',
+    'spawn_agents',
+    'suggest_followups',
+    'thinker_gemini',
+  ),
+  // helper agent
+  tools('add_message', 'read_files', 'run_terminal_command', 'set_output'),
+]
+
+/** Toolsets observed proxying our free endpoint, by harness. */
+const FOREIGN_TOOLSETS: Array<[string, ReturnType<typeof tools>]> = [
+  ['claude-code', tools('Bash', 'Edit', 'Glob', 'Grep', 'Read', 'Write')],
+  [
+    'opencode',
+    tools(
+      'ask',
+      'bash',
+      'edit',
+      'eval',
+      'glob',
+      'grep',
+      'hub',
+      'read',
+      'task',
+      'todo',
+      'web_search',
+      'write',
+    ),
+  ],
+  [
+    'cline',
+    tools(
+      'list_files',
+      'read_file',
+      'replace_in_file',
+      'search_files',
+      'write_file',
+    ),
+  ],
+  [
+    'novel-farm',
+    tools(
+      'check_consistency',
+      'commit_chapter',
+      'draft_chapter',
+      'edit_chapter',
+      'novel_context',
+      'plan_chapter',
+      'read_chapter',
+    ),
+  ],
+  [
+    'pentest-harness',
+    tools(
+      'analyze_target_graph',
+      'delegate_task',
+      'edit_source_code',
+      'execute_command',
+      'install_tool',
+      'python_execute',
+      'read_file',
+    ),
+  ],
+]
+
+describe('detectForeignAnvilClient', () => {
+  test('the signature is every non-generic tool we define', () => {
+    // Derived, not hand-listed: a tool added to `toolNames` joins the signature
+    // automatically. That is the rot that flagged researcher-web — a
+    // hand-picked list simply never grew to cover it.
+    const known = new Set<string>(toolNames)
+    for (const name of known) {
+      expect(ANVIL_SIGNATURE_TOOL_NAMES.has(name)).toBe(
+        !GENERIC_TOOL_NAMES.has(name),
+      )
+    }
+    // Every generic name must be one we actually define, or it is dead weight.
+    for (const name of GENERIC_TOOL_NAMES) {
+      expect(known.has(name)).toBe(true)
+    }
+    expect(ANVIL_SIGNATURE_TOOL_NAMES.size).toBeGreaterThan(20)
+  })
+
+  test('clears real anvil toolsets', () => {
+    for (const toolset of ANVIL_TOOLSETS) {
+      expect(detectForeignAnvilClient({ tools: toolset }).signal).toBeNull()
+    }
+  })
+
+  test.each(FOREIGN_TOOLSETS)('flags %s', (_name, toolset) => {
+    expect(detectForeignAnvilClient({ tools: toolset }).signal).toBe(
+      'foreign_toolset',
+    )
+  })
+
+  test('sharing a few generic names does not launder a foreign harness', () => {
+    // opencode sends `glob` and `web_search`, which we define — but both are
+    // generic, so neither is in the signature and the overlap buys it nothing.
+    expect(
+      detectForeignAnvilClient({
+        tools: tools('glob', 'web_search', 'bash', 'edit', 'write'),
+      }).signal,
+    ).toBe('foreign_toolset')
+  })
+
+  test('a toolset of only generic names is foreign', () => {
+    // Measured over 30 days: 406 users on spoofed `base2-free-*` agent ids send
+    // a bare `web_search` and nothing else. No agent we ship has a toolset of
+    // only generic names — every single-tool agent of ours uses a distinctive
+    // one (`run_terminal_command`, `spawn_agents`, `read_docs`, `set_output`).
+    expect(
+      detectForeignAnvilClient({ tools: tools('web_search') }).signal,
+    ).toBe('foreign_toolset')
+    expect(
+      detectForeignAnvilClient({ tools: tools('glob', 'web_search') }).signal,
+    ).toBe('foreign_toolset')
+  })
+
+  test('our toolset wins over sampling params', () => {
+    // 16 users in one day sent our tools AND set params. Downgrading them is
+    // the false positive this ordering exists to prevent.
+    expect(
+      detectForeignAnvilClient({
+        tools: tools('ask_user', 'read_files'),
+        temperature: 0.3,
+        max_tokens: 32000,
+      }).signal,
+    ).toBeNull()
+  })
+
+  test('flags sampling params only when no tools are offered', () => {
+    expect(detectForeignAnvilClient({ temperature: 0.7 }).signal).toBe(
+      'sampling_params',
+    )
+    expect(detectForeignAnvilClient({ top_p: 0.9 }).signal).toBe(
+      'sampling_params',
+    )
+    expect(detectForeignAnvilClient({ max_tokens: 4096 }).signal).toBe(
+      'sampling_params',
+    )
+    expect(
+      detectForeignAnvilClient({ max_completion_tokens: 4096 }).signal,
+    ).toBe('sampling_params')
+  })
+
+  test('clears a tool-free request that leaves sampling params unset', () => {
+    // Our helper agents (chat titles, compaction) send no tools at all.
+    expect(detectForeignAnvilClient({}).signal).toBeNull()
+    expect(detectForeignAnvilClient({ tools: [] }).signal).toBeNull()
+  })
+
+  test('explicit nulls are not treated as set', () => {
+    // Regression: this test used to pass `undefined` while claiming to cover
+    // `null`, so it passed against a detector that flagged every explicit
+    // null. A client that serializes its whole body sends `temperature: null`
+    // rather than omitting the key, and that is unset.
+    for (const body of [
+      { temperature: null },
+      { top_p: null },
+      { max_tokens: null },
+      { max_completion_tokens: null },
+      { temperature: null, top_p: null, max_tokens: null },
+    ]) {
+      expect(detectForeignAnvilClient(body as never).signal).toBeNull()
+    }
+    expect(
+      detectForeignAnvilClient({
+        temperature: undefined,
+        top_p: undefined,
+        max_tokens: undefined,
+      }).signal,
+    ).toBeNull()
+  })
+
+  test('zero is a real choice and stays flagged', () => {
+    // `!= null` must not swallow falsy-but-set values.
+    expect(detectForeignAnvilClient({ temperature: 0 }).signal).toBe(
+      'sampling_params',
+    )
+    expect(detectForeignAnvilClient({ top_p: 0 }).signal).toBe(
+      'sampling_params',
+    )
+  })
+
+  test('tolerates malformed tool entries without throwing', () => {
+    for (const tools of [null, undefined, 'nope', [], [null], [{}], [{ function: {} }]]) {
+      expect(() =>
+        detectForeignAnvilClient({ tools } as never),
+      ).not.toThrow()
+    }
+    // Tools present but unparseable read as "no tools offered", so the request
+    // falls through to the param check rather than being flagged on a name
+    // list we could not actually read.
+    expect(detectForeignAnvilClient({ tools: [{}] }).signal).toBeNull()
+  })
+
+  test('truncates caller-controlled tool names before they reach logs', () => {
+    const verdict = detectForeignAnvilClient({
+      tools: [{ type: 'function', function: { name: 'x'.repeat(5000) } }],
+    })
+    expect(verdict.signal).toBe('foreign_toolset')
+    expect(verdict.sampleToolNames[0]!.length).toBeLessThanOrEqual(64)
+  })
+
+  test('reports bounded evidence for the log line', () => {
+    const verdict = detectForeignAnvilClient({
+      tools: tools('a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j'),
+    })
+    expect(verdict.toolCount).toBe(10)
+    expect(verdict.sampleToolNames).toHaveLength(8)
+  })
+
+  test.each([
+    ['researcher-web', ['web_search', 'read_url']],
+    ['researcher-docs', ['read_docs']],
+    ['anvil-desktop-autorun', ['decide']],
+    ['basher', ['run_terminal_command']],
+    ['file-picker', ['spawn_agents']],
+  ])('clears our own %s toolset', (_agent, names) => {
+    // Backtested over 30 days against the signature alone: researcher-web was
+    // flagged on 100% of 334,042 requests from 4,821 users, autorun on 100% of
+    // 2,904 from 41. `web_search` cannot join the signature (opencode ships
+    // it), so these clear by the every-tool-is-ours rule instead.
+    expect(detectForeignAnvilClient({ tools: tools(...names) }).signal).toBeNull()
+  })
+
+  test('borrowing one distinctive name clears an otherwise foreign toolset', () => {
+    // Known and accepted, not an oversight. `some` semantics are what let an
+    // MCP user attach `ghidra__*` alongside our tools without being flagged,
+    // and the cost is that a proxy declaring one of our names is cleared too.
+    // Self-limiting: the borrowed name becomes a GENERIC_TOOL_NAMES candidate
+    // the moment it shows up in the logs, and the proxy has to actually
+    // implement the tool for its own loop to keep working.
+    expect(
+      detectForeignAnvilClient({ tools: tools('read_files', 'Bash') }).signal,
+    ).toBeNull()
+  })
+
+  test('a root agent offering no tools is a bare completion proxy', () => {
+    // Our roots always ship their toolset — that is what makes them agentic.
+    // Measured over 7 days with assistant-response rows excluded, the desktop
+    // roots send zero tool-free requests (0 of 683,151 for -v3, 0 of 294,823
+    // for -worktree) and the CLI roots 0.30%/2.81%. All 18 users sampled across
+    // that tail were non-coding automation.
+    expect(detectForeignAnvilClient({}, true).signal).toBe(
+      'root_agent_no_tools',
+    )
+    // Sampling params do not change the verdict for a root.
+    expect(detectForeignAnvilClient({ temperature: 0.7 }, true).signal).toBe(
+      'root_agent_no_tools',
+    )
+  })
+
+  test('a root agent sending our tools is still ours', () => {
+    // Evading root_agent_no_tools means sending our toolset, at which point the
+    // toolset check applies instead — the same convergent property.
+    expect(
+      detectForeignAnvilClient({ tools: tools('ask_user') }, true).signal,
+    ).toBeNull()
+    expect(
+      detectForeignAnvilClient({ tools: tools('Bash', 'Edit') }, true).signal,
+    ).toBe('foreign_toolset')
+  })
+
+  test('a tool-free SUBagent is untouched', () => {
+    // Our helper agents (chat titles, compaction, researcher-docs) legitimately
+    // send no tools; only ROOT agents are agentic by definition. Defaulting
+    // isRootAgent to false keeps every non-root caller on the old behaviour.
+    expect(detectForeignAnvilClient({}).signal).toBeNull()
+    expect(detectForeignAnvilClient({}, false).signal).toBeNull()
+  })
+
+  test('downgrade target is the free OpenRouter variant', () => {
+    expect(ANVIL_DOWNGRADE_MODEL_ID).toBe('inclusionai/ling-3.0-tiny:free')
+    expect(ANVIL_DOWNGRADE_MODEL_ID.endsWith(':free')).toBe(true)
+  })
+})
+
+describe('resolveForeignClientDowngrade', () => {
+  const foreign = { tools: tools('Bash', 'Edit') }
+  const params = { max_completion_tokens: 977_725 }
+  const ours = { tools: tools('ask_user', 'read_files') }
+
+  test('always downgrades a foreign toolset', () => {
+    // Third-party clients are a terms violation: Anvil funds free inference
+    // with ads only our own clients render, so a proxied request takes the
+    // cost and returns none of the revenue. There is no mode in which this is
+    // served what it asked for.
+    expect(resolveForeignClientDowngrade({ body: foreign })!.downgradeTo).toBe(
+      ANVIL_DOWNGRADE_MODEL_ID,
+    )
+  })
+
+  test('reports but never acts on a tool-free root agent', () => {
+    // Deliberately report-only. The 30-day backtest found 3,729 users who mix
+    // tool-free root requests into real agentic traffic, 999 of whose sessions
+    // contain both — enforcing per-request swaps the model mid-session for real
+    // coding runs. Only 417 users are pure proxies, and no run-length threshold
+    // separates them (the MIXED cohort holds the longest tool-free run, 11,094,
+    // vs 2,153 for the proxies). Flipping this to enforce needs an
+    // account-level verdict, not a change here.
+    const d = resolveForeignClientDowngrade({ body: {}, isRootAgent: true })!
+    expect(d.signal).toBe('root_agent_no_tools')
+    expect(d.downgradeTo).toBeNull()
+  })
+
+  test('leaves a tool-free non-root request alone', () => {
+    expect(resolveForeignClientDowngrade({ body: {} })).toBeNull()
+  })
+
+  test('reports but never acts on the sampling-param signal', () => {
+    const decision = resolveForeignClientDowngrade({ body: params })!
+    expect(decision.signal).toBe('sampling_params')
+    expect(decision.downgradeTo).toBeNull()
+  })
+
+  test('a anvil toolset is never reported', () => {
+    expect(resolveForeignClientDowngrade({ body: ours })).toBeNull()
+  })
+
+  test('does not re-downgrade a request already on the downgrade model', () => {
+    const decision = resolveForeignClientDowngrade({
+      body: { ...foreign, model: ANVIL_DOWNGRADE_MODEL_ID },
+    })!
+    expect(decision.signal).toBe('foreign_toolset')
+    expect(decision.downgradeTo).toBeNull()
+  })
+})