@ic-reactor/mcp 0.0.0-stage → 0.0.0

package/evals/harness/auth.mjs

@@ -0,0 +1,91 @@
+// How an agent run authenticates, and keeping the credential out of results.
+//
+// Three sources, in this order of precedence:
+// 1. `--oauth-token-file <path>`: a file holding a Claude subscription token
+//    (`claude setup-token`), trimmed. Refused if group- or world-readable.
+// 2. CLAUDE_CODE_OAUTH_TOKEN in the driver's environment: the same kind of
+//    token. It works for headless `claude -p` with no config directory (and
+//    is not read in `--bare` mode, which the driver therefore never uses).
+// 3. ANTHROPIC_API_KEY: API billing. The CLI prefers an API key over an OAuth
+//    token when both are set, so the driver passes ONLY the chosen
+//    credential: with a token available, the subscription is what gets used.
+// A token from the file reaches the agent as CLAUDE_CODE_OAUTH_TOKEN. The
+// value is never printed or recorded: everything the driver writes goes
+// through `redact`, which removes every credential value it knows of — the
+// environment's and any read from a file — wherever it appears (an agent can
+// read its own environment and echo it into a transcript or a file).
+import { readFileSync, statSync } from "node:fs"
+
+export const AUTH_VARS = {
+  oauth: "CLAUDE_CODE_OAUTH_TOKEN",
+  api_key: "ANTHROPIC_API_KEY",
+}
+
+// Credential values read from files, so `redact` knows them too.
+const registered = new Set()
+
+/** Reads a token file, refusing one that anyone but its owner can read. */
+export function readTokenFile(path) {
+  const mode = statSync(path).mode
+  if ((mode & 0o077) !== 0) {
+    throw new Error(
+      `${path} is readable or writable by group or others (mode ${(mode & 0o777).toString(8)}); ` +
+        `run \`chmod 600 ${path}\``
+    )
+  }
+  // A token has no whitespace; one copied from a terminal often carries the
+  // line break the terminal wrapped it at, so drop all of it, not just the ends.
+  const value = readFileSync(path, "utf8").replace(/\s+/g, "")
+  if (value === "") throw new Error(`${path} is empty`)
+  registered.add(value)
+  return value
+}
+
+/**
+ * `{ kind, name, value, source }` for the credential to use, or `undefined`.
+ * `name` is the variable it is passed to the agent as.
+ */
+export function resolveAuth(env = process.env, { tokenFile } = {}) {
+  if (tokenFile) {
+    return {
+      kind: "oauth",
+      name: AUTH_VARS.oauth,
+      value: readTokenFile(tokenFile),
+      source: "file",
+    }
+  }
+  for (const kind of ["oauth", "api_key"]) {
+    const name = AUTH_VARS[kind]
+    const value = env[name]
+    if (typeof value === "string" && value.trim() !== "") {
+      return { kind, name, value: value.trim(), source: "env" }
+    }
+  }
+  return undefined
+}
+
+export const MISSING_AUTH_MESSAGE =
+  "no agent credential: pass --oauth-token-file <path> or set " +
+  "CLAUDE_CODE_OAUTH_TOKEN (a Claude subscription token; create one with " +
+  "`claude setup-token`), or set ANTHROPIC_API_KEY (API billing). The agent " +
+  "runs with a clean CLAUDE_CONFIG_DIR, so the host's login is not available to it."
+
+/** Every credential value known (environment and files), longest first. */
+export function secretsOf(env = process.env) {
+  return [
+    ...new Set([
+      ...Object.values(AUTH_VARS).map((name) => env[name]),
+      ...Object.values(AUTH_VARS).map((name) => env[name]?.trim()),
+      ...registered,
+    ]),
+  ]
+    .filter((v) => typeof v === "string" && v.length >= 8)
+    .sort((a, b) => b.length - a.length)
+}
+
+/** `text` with every credential value replaced by a marker. */
+export function redact(text, secrets = secretsOf()) {
+  let out = String(text)
+  for (const secret of secrets) out = out.split(secret).join("[REDACTED]")
+  return out
+}

package/evals/harness/check-docs.mjs

@@ -0,0 +1,157 @@
+#!/usr/bin/env node
+// Checks that condition docs do not leak the hidden tests: no test name (as
+// written, or with spaces for underscores), and no distinctive literal from a
+// hidden test file that the task prompts do not already make public.
+//
+//   node harness/check-docs.mjs [doc files...]   (default: the three guides: v4-proto, thin-guide, v4)
+//
+// "Distinctive" = a string or numeric literal from a hidden test that holds a
+// digit, `_` or `.`, is at least 4 characters, is not an import specifier, and
+// does not appear in the task prompts of EVERY prompt variant (a literal only
+// the explicit prompt states is still a needle: the docs are the same under
+// the minimal prompt, where it is not public). Paraphrase cannot be checked
+// mechanically; README records the manual review.
+import { readFileSync, readdirSync } from "node:fs"
+import { join, relative } from "node:path"
+import {
+  EVALS,
+  PROMPT_VARIANTS,
+  TASKS,
+  promptFile,
+  taskSpec,
+  taskTests,
+} from "./assemble.mjs"
+
+const DEFAULT_DOCS = [
+  "conditions/v4-proto/docs/llms.txt",
+  "conditions/thin-guide/docs/llms.txt",
+  // Copied from the packed @ic-reactor/core by setup.mjs, which refuses it on
+  // a hit before copying; listed here so the default run covers it too.
+  "conditions/v4/docs/llms.txt",
+]
+
+export function hiddenNeedles() {
+  // One text per variant: all tasks' prompts of that variant.
+  const promptsByVariant = PROMPT_VARIANTS.map((variant) =>
+    TASKS.map((t) => readFileSync(promptFile(t, variant), "utf8")).join("\n")
+  )
+  const isPublic = (text) =>
+    promptsByVariant.every((prompts) => prompts.includes(text))
+  const needles = new Map() // needle → where it came from
+  for (const task of TASKS) {
+    // Test names from task.json, the authoritative list (the files build
+    // some names from templates).
+    for (const name of taskTests(taskSpec(task))) {
+      needles.set(name, `${task}/task.json (test name)`)
+      needles.set(
+        name.replaceAll("_", " "),
+        `${task}/task.json (test name, spaced)`
+      )
+    }
+    const dir = join(EVALS, "tasks", task, "hidden")
+    for (const file of readdirSync(dir)) {
+      const source = readFileSync(join(dir, file), "utf8")
+      const where = `${task}/hidden/${file}`
+
+      const code = source
+        .split("\n")
+        .filter((line) => !/^\s*(import|\}\s*from)\b/.test(line))
+        .join("\n")
+        // Test-runner timeouts are not assertions: `}, 30_000)`, `timeout: 20_000`.
+        .replace(/\}\s*,\s*[\d_]+\s*\)/g, "})")
+        .replace(/timeout:\s*[\d_]+/g, "timeout: 0")
+        .replace(/timeout\s*=\s*[\d_]+/g, "timeout = 0")
+        // Nor are waits: `await sleep(1_000)`.
+        .replace(/sleep\(\s*[\d_]+\s*\)/g, "sleep(0)")
+      const literals = [
+        ...[...code.matchAll(/"((?:[^"\\\n]|\\.)*)"/g)].map((m) => m[1]),
+        ...[...code.matchAll(/`([^`$\n]*)`/g)].map((m) => m[1]),
+        ...[...code.matchAll(/\b\d[\d_]{3,}n?\b/g)].map((m) => m[0]),
+      ]
+      for (const literal of literals) {
+        const numeric = /^\d[\d_]*n?$/.test(literal)
+        const plain = numeric
+          ? literal.replace(/_/g, "").replace(/n$/, "")
+          : literal
+        if (literal.length < 4 || !/[\d_.]/.test(literal)) continue
+        if (/^[@#.]/.test(literal)) continue
+        if (isPublic(literal) || isPublic(plain)) continue
+        needles.set(literal, `${where} (literal)`)
+        if (plain !== literal && plain.length >= 4)
+          needles.set(plain, `${where} (literal)`)
+      }
+    }
+  }
+  return needles
+}
+
+export function checkDocs(files) {
+  const needles = hiddenNeedles()
+  const hits = []
+  for (const file of files) {
+    const text = readFileSync(file, "utf8").toLowerCase()
+    for (const [needle, where] of needles) {
+      if (text.includes(needle.toLowerCase())) {
+        hits.push({ file: relative(EVALS, file), needle, from: where })
+      }
+    }
+  }
+  return { needles: needles.size, hits }
+}
+
+/**
+ * Every hidden-test name, from task.json (the authoritative list), as
+ * written and with spaces for underscores. Taken from task.json rather than
+ * from hiddenNeedles(), where a name that is also a string literal in a test
+ * file is recorded under the file it was last seen in.
+ */
+export function hiddenTestNames() {
+  return TASKS.flatMap((task) =>
+    taskTests(taskSpec(task)).flatMap((name) => [
+      { needle: name, from: `${task}/task.json (test name)` },
+      {
+        needle: name.replaceAll("_", " "),
+        from: `${task}/task.json (test name, spaced)`,
+      },
+    ])
+  )
+}
+
+/**
+ * The prompts of every variant: no hidden-test name (as written or spaced).
+ * Literals are not checked here: the prompts are what makes a literal public.
+ */
+export function checkPrompts() {
+  const names = hiddenTestNames()
+  const hits = []
+  for (const task of TASKS) {
+    for (const variant of PROMPT_VARIANTS) {
+      const file = promptFile(task, variant)
+      const text = readFileSync(file, "utf8").toLowerCase()
+      for (const { needle, from } of names) {
+        if (text.includes(needle.toLowerCase())) {
+          hits.push({ file: relative(EVALS, file), needle, from })
+        }
+      }
+    }
+  }
+  return { prompts: TASKS.length * PROMPT_VARIANTS.length, hits }
+}
+
+if (import.meta.url === `file://${process.argv[1]}`) {
+  const files = (
+    process.argv.length > 2 ? process.argv.slice(2) : DEFAULT_DOCS
+  ).map((f) => (f.startsWith("/") ? f : join(EVALS, f)))
+  const { needles, hits } = checkDocs(files)
+  const prompts = checkPrompts()
+  process.stdout.write(
+    `check-docs: ${files.length} file(s), ${needles} hidden-test needles, ${hits.length} hit(s); ` +
+      `${prompts.prompts} prompt(s) (${PROMPT_VARIANTS.join(", ")}), ${prompts.hits.length} test-name hit(s)\n`
+  )
+  for (const hit of [...hits, ...prompts.hits]) {
+    process.stdout.write(
+      `  ${hit.file}: ${JSON.stringify(hit.needle)} from ${hit.from}\n`
+    )
+  }
+  process.exit(hits.length + prompts.hits.length === 0 ? 0 : 1)
+}

package/evals/harness/declarations/icrc1.did.d.ts

@@ -0,0 +1,95 @@
+/* eslint-disable */
+
+// @ts-nocheck
+
+// This file was automatically generated by @icp-sdk/bindgen@0.2.1.
+// You should NOT make any changes in this file as it will be overwritten.
+// Additionally, you should also exclude this file from your linter and/or formatter to prevent it from being checked or modified.
+
+import type { ActorMethod } from "@icp-sdk/core/agent"
+import type { IDL } from "@icp-sdk/core/candid"
+import type { Principal } from "@icp-sdk/core/principal"
+
+export interface Account {
+  owner: Principal
+  subaccount: [] | [Subaccount]
+}
+export type BlockIndex = bigint
+/**
+ * Number of nanoseconds between two [Timestamp]s.
+ */
+export type Duration = bigint
+/**
+ * The initialization parameters of the Ledger
+ */
+export interface InitArgs {
+  token_symbol: string
+  transfer_fee: bigint
+  metadata: Array<[string, Value]>
+  minting_account: Account
+  initial_balances: Array<[Account, bigint]>
+  archive_options: {
+    num_blocks_to_archive: bigint
+    trigger_threshold: bigint
+    max_message_size_bytes: [] | [bigint]
+    cycles_for_archive_creation: [] | [bigint]
+    node_max_memory_size_bytes: [] | [bigint]
+    controller_id: Principal
+  }
+  token_name: string
+}
+export type LedgerArg = { Upgrade: [] | [UpgradeArgs] } | { Init: InitArgs }
+export type Subaccount = Uint8Array
+/**
+ * Number of nanoseconds since the UNIX epoch in UTC timezone.
+ */
+export type Timestamp = bigint
+export type Tokens = bigint
+export interface TransferArg {
+  to: Account
+  fee: [] | [Tokens]
+  memo: [] | [Uint8Array]
+  from_subaccount: [] | [Subaccount]
+  created_at_time: [] | [Timestamp]
+  amount: Tokens
+}
+export type TransferError =
+  | {
+      GenericError: { message: string; error_code: bigint }
+    }
+  | { TemporarilyUnavailable: null }
+  | { BadBurn: { min_burn_amount: Tokens } }
+  | { Duplicate: { duplicate_of: BlockIndex } }
+  | { BadFee: { expected_fee: Tokens } }
+  | { CreatedInFuture: { ledger_time: bigint } }
+  | { TooOld: null }
+  | { InsufficientFunds: { balance: Tokens } }
+export type TransferResult = { Ok: BlockIndex } | { Err: TransferError }
+export interface UpgradeArgs {
+  token_symbol: [] | [string]
+  transfer_fee: [] | [bigint]
+  metadata: [] | [Array<[string, Value]>]
+  token_name: [] | [string]
+}
+/**
+ * The value returned from the [icrc1_metadata] endpoint.
+ */
+export type Value =
+  { Int: bigint } | { Nat: bigint } | { Blob: Uint8Array } | { Text: string }
+export interface _SERVICE {
+  icrc1_balance_of: ActorMethod<[Account], Tokens>
+  icrc1_decimals: ActorMethod<[], number>
+  icrc1_fee: ActorMethod<[], Tokens>
+  icrc1_metadata: ActorMethod<[], Array<[string, Value]>>
+  icrc1_minting_account: ActorMethod<[], [] | [Account]>
+  icrc1_name: ActorMethod<[], string>
+  icrc1_supported_standards: ActorMethod<
+    [],
+    Array<{ url: string; name: string }>
+  >
+  icrc1_symbol: ActorMethod<[], string>
+  icrc1_total_supply: ActorMethod<[], Tokens>
+  icrc1_transfer: ActorMethod<[TransferArg], TransferResult>
+}
+export declare const idlFactory: IDL.InterfaceFactory
+export declare const init: (args: { IDL: typeof IDL }) => IDL.Type[]

package/evals/harness/declarations/icrc1.did.js

@@ -0,0 +1,140 @@
+/* eslint-disable */
+
+// @ts-nocheck
+
+// This file was automatically generated by @icp-sdk/bindgen@0.2.1.
+// You should NOT make any changes in this file as it will be overwritten.
+// Additionally, you should also exclude this file from your linter and/or formatter to prevent it from being checked or modified.
+
+import { IDL } from "@icp-sdk/core/candid"
+
+export const idlFactory = ({ IDL }) => {
+  const Value = IDL.Variant({
+    Int: IDL.Int,
+    Nat: IDL.Nat,
+    Blob: IDL.Vec(IDL.Nat8),
+    Text: IDL.Text,
+  })
+  const UpgradeArgs = IDL.Record({
+    token_symbol: IDL.Opt(IDL.Text),
+    transfer_fee: IDL.Opt(IDL.Nat64),
+    metadata: IDL.Opt(IDL.Vec(IDL.Tuple(IDL.Text, Value))),
+    token_name: IDL.Opt(IDL.Text),
+  })
+  const Subaccount = IDL.Vec(IDL.Nat8)
+  const Account = IDL.Record({
+    owner: IDL.Principal,
+    subaccount: IDL.Opt(Subaccount),
+  })
+  const InitArgs = IDL.Record({
+    token_symbol: IDL.Text,
+    transfer_fee: IDL.Nat64,
+    metadata: IDL.Vec(IDL.Tuple(IDL.Text, Value)),
+    minting_account: Account,
+    initial_balances: IDL.Vec(IDL.Tuple(Account, IDL.Nat64)),
+    archive_options: IDL.Record({
+      num_blocks_to_archive: IDL.Nat64,
+      trigger_threshold: IDL.Nat64,
+      max_message_size_bytes: IDL.Opt(IDL.Nat64),
+      cycles_for_archive_creation: IDL.Opt(IDL.Nat64),
+      node_max_memory_size_bytes: IDL.Opt(IDL.Nat64),
+      controller_id: IDL.Principal,
+    }),
+    token_name: IDL.Text,
+  })
+  const LedgerArg = IDL.Variant({
+    Upgrade: IDL.Opt(UpgradeArgs),
+    Init: InitArgs,
+  })
+  const Tokens = IDL.Nat
+  const Timestamp = IDL.Nat64
+  const TransferArg = IDL.Record({
+    to: Account,
+    fee: IDL.Opt(Tokens),
+    memo: IDL.Opt(IDL.Vec(IDL.Nat8)),
+    from_subaccount: IDL.Opt(Subaccount),
+    created_at_time: IDL.Opt(Timestamp),
+    amount: Tokens,
+  })
+  const BlockIndex = IDL.Nat
+  const TransferError = IDL.Variant({
+    GenericError: IDL.Record({
+      message: IDL.Text,
+      error_code: IDL.Nat,
+    }),
+    TemporarilyUnavailable: IDL.Null,
+    BadBurn: IDL.Record({ min_burn_amount: Tokens }),
+    Duplicate: IDL.Record({ duplicate_of: BlockIndex }),
+    BadFee: IDL.Record({ expected_fee: Tokens }),
+    CreatedInFuture: IDL.Record({ ledger_time: IDL.Nat64 }),
+    TooOld: IDL.Null,
+    InsufficientFunds: IDL.Record({ balance: Tokens }),
+  })
+  const TransferResult = IDL.Variant({
+    Ok: BlockIndex,
+    Err: TransferError,
+  })
+
+  return IDL.Service({
+    icrc1_balance_of: IDL.Func([Account], [Tokens], ["query"]),
+    icrc1_decimals: IDL.Func([], [IDL.Nat8], ["query"]),
+    icrc1_fee: IDL.Func([], [Tokens], ["query"]),
+    icrc1_metadata: IDL.Func(
+      [],
+      [IDL.Vec(IDL.Tuple(IDL.Text, Value))],
+      ["query"]
+    ),
+    icrc1_minting_account: IDL.Func([], [IDL.Opt(Account)], ["query"]),
+    icrc1_name: IDL.Func([], [IDL.Text], ["query"]),
+    icrc1_supported_standards: IDL.Func(
+      [],
+      [IDL.Vec(IDL.Record({ url: IDL.Text, name: IDL.Text }))],
+      ["query"]
+    ),
+    icrc1_symbol: IDL.Func([], [IDL.Text], ["query"]),
+    icrc1_total_supply: IDL.Func([], [Tokens], ["query"]),
+    icrc1_transfer: IDL.Func([TransferArg], [TransferResult], []),
+  })
+}
+
+export const init = ({ IDL }) => {
+  const Value = IDL.Variant({
+    Int: IDL.Int,
+    Nat: IDL.Nat,
+    Blob: IDL.Vec(IDL.Nat8),
+    Text: IDL.Text,
+  })
+  const UpgradeArgs = IDL.Record({
+    token_symbol: IDL.Opt(IDL.Text),
+    transfer_fee: IDL.Opt(IDL.Nat64),
+    metadata: IDL.Opt(IDL.Vec(IDL.Tuple(IDL.Text, Value))),
+    token_name: IDL.Opt(IDL.Text),
+  })
+  const Subaccount = IDL.Vec(IDL.Nat8)
+  const Account = IDL.Record({
+    owner: IDL.Principal,
+    subaccount: IDL.Opt(Subaccount),
+  })
+  const InitArgs = IDL.Record({
+    token_symbol: IDL.Text,
+    transfer_fee: IDL.Nat64,
+    metadata: IDL.Vec(IDL.Tuple(IDL.Text, Value)),
+    minting_account: Account,
+    initial_balances: IDL.Vec(IDL.Tuple(Account, IDL.Nat64)),
+    archive_options: IDL.Record({
+      num_blocks_to_archive: IDL.Nat64,
+      trigger_threshold: IDL.Nat64,
+      max_message_size_bytes: IDL.Opt(IDL.Nat64),
+      cycles_for_archive_creation: IDL.Opt(IDL.Nat64),
+      node_max_memory_size_bytes: IDL.Opt(IDL.Nat64),
+      controller_id: IDL.Principal,
+    }),
+    token_name: IDL.Text,
+  })
+  const LedgerArg = IDL.Variant({
+    Upgrade: IDL.Opt(UpgradeArgs),
+    Init: InitArgs,
+  })
+
+  return [LedgerArg]
+}