@tangle-network/agent-bench 0.3.6 → 0.3.8

package/src/swe-arena/activation.mts

@@ -0,0 +1,228 @@
+/**
+ * Gen-5 activation gate (SOTA adoption #2, GSME-style "verify the mechanism
+ * fired before the score counts").
+ *
+ * Every proposer's deliverable must include a MACHINE-CHECKABLE ACTIVATION
+ * PREDICATE at `.improve/activation.json` (inside the change-space's metadata
+ * prefix, so the improvement driver's finalize commits it with the candidate):
+ * a grep pattern or script over the candidate's OWN campaign run artifacts
+ * that proves its mechanism actually fired (e.g. "the new prompt section
+ * rendered in worker prompts", "patchRiskWarnings emitted in >=1 settle").
+ *
+ * Enforcement is two-stage, both fail-closed:
+ *   1. PREFILTER — a candidate without a parseable predicate is killed before
+ *      any evaluation spend (proposer-fanout.mts, stage 'activation-predicate').
+ *   2. POST-EVAL — the evaluator runs the predicate over the candidate's own
+ *      cell run dirs; a candidate whose mechanism NEVER fired is QUARANTINED
+ *      (staircase verdict 'quarantined-inactive': recorded, never promoted)
+ *      even when its score improved — a score with an inactive mechanism is
+ *      indistinguishable from luck or from gaming the visible instances.
+ */
+
+import { readFile, readdir, stat } from 'node:fs/promises'
+import { join, relative } from 'node:path'
+import { run } from './proc.ts'
+
+export const ACTIVATION_PREDICATE_RELPATH = '.improve/activation.json'
+
+export interface ActivationPredicate {
+  version: 'v1'
+  /** One sentence: which mechanism this proves fired. */
+  description: string
+  kind: 'grep' | 'script'
+  /** kind 'grep': JS RegExp source tested line-by-line over run artifacts. */
+  pattern?: string
+  /** kind 'grep': optional relative-path substring filters (a file is searched
+   *  when its run-dir-relative path contains ANY entry). Empty/absent = all. */
+  files?: string[]
+  /** kind 'script': bash script; run once per run dir with cwd=<runDir> and
+   *  $RUN_DIR set; exit 0 in ANY run dir = mechanism fired. */
+  script?: string
+}
+
+export type ParsedPredicate = { ok: true; predicate: ActivationPredicate } | { ok: false; error: string }
+
+export function parseActivationPredicate(raw: string): ParsedPredicate {
+  let value: unknown
+  try {
+    value = JSON.parse(raw)
+  } catch (cause) {
+    return { ok: false, error: `not valid JSON: ${(cause as Error).message}` }
+  }
+  if (typeof value !== 'object' || value === null || Array.isArray(value)) {
+    return { ok: false, error: 'must be a JSON object' }
+  }
+  const p = value as Record<string, unknown>
+  if (p.version !== 'v1') return { ok: false, error: `version must be "v1", got ${JSON.stringify(p.version)}` }
+  if (typeof p.description !== 'string' || p.description.trim().length === 0) {
+    return { ok: false, error: 'description must be a non-empty string' }
+  }
+  if (p.kind === 'grep') {
+    if (typeof p.pattern !== 'string' || p.pattern.length === 0) {
+      return { ok: false, error: 'kind "grep" requires a non-empty pattern' }
+    }
+    try {
+      new RegExp(p.pattern)
+    } catch (cause) {
+      return { ok: false, error: `pattern is not a valid RegExp: ${(cause as Error).message}` }
+    }
+    if (p.files !== undefined && (!Array.isArray(p.files) || p.files.some((f) => typeof f !== 'string'))) {
+      return { ok: false, error: 'files must be an array of strings when present' }
+    }
+  } else if (p.kind === 'script') {
+    if (typeof p.script !== 'string' || p.script.trim().length === 0) {
+      return { ok: false, error: 'kind "script" requires a non-empty script' }
+    }
+  } else {
+    return { ok: false, error: `kind must be "grep" or "script", got ${JSON.stringify(p.kind)}` }
+  }
+  return { ok: true, predicate: p as unknown as ActivationPredicate }
+}
+
+/** The prompt-visible contract: template + worked example. */
+export function activationPredicateInstruction(): string {
+  return [
+    'ACTIVATION PREDICATE (required deliverable — a candidate without one is rejected before evaluation):',
+    `Write ${ACTIVATION_PREDICATE_RELPATH} in this worktree: a machine-checkable proof that YOUR mechanism`,
+    "actually fired during evaluation. The evaluator runs it over your candidate's own run artifacts",
+    '(each arm run dir: driver.log, brain.jsonl, result.json, ws/.loops/** journal + worker evidence);',
+    'if it never fires, your candidate is QUARANTINED even when its score improved.',
+    '',
+    'Template (kind "grep" — a RegExp tested over the run artifacts):',
+    '  {',
+    '    "version": "v1",',
+    '    "description": "patchRiskWarnings emitted in at least one settle",',
+    '    "kind": "grep",',
+    '    "pattern": "patchRiskWarnings|patch-risk",',
+    '    "files": ["journal.jsonl", "workers/"]',
+    '  }',
+    'Or kind "script": {"version":"v1","description":"...","kind":"script","script":"grep -rq NEW_SECTION ws/.loops"}',
+    '(exit 0 in any run dir = fired).',
+    'Pick a pattern that can ONLY appear when your mechanism ran — not one that matches the diff itself.',
+  ].join('\n')
+}
+
+// ---------------------------------------------------------------------------
+// Predicate execution over run dirs.
+// ---------------------------------------------------------------------------
+
+export interface ActivationResult {
+  fired: boolean
+  /** Bounded evidence: matching `path:line` refs (grep) or script stdout tails. */
+  evidence: string[]
+  checkedRunDirs: number
+  checkedFiles: number
+  /** Bounded notes on skipped inputs (oversized files, walk caps). */
+  warnings: string[]
+}
+
+const MAX_FILES_PER_RUN_DIR = 4000
+const MAX_FILE_BYTES = 32 * 1024 * 1024
+const MAX_EVIDENCE = 10
+const SCRIPT_TIMEOUT_MS = 120_000
+
+/** Walk one run dir. The `ws/` workspace subtree (a whole checked-out repo) is
+ *  skipped EXCEPT `ws/.loops/**` — the supervisor's own artifacts live there
+ *  and are exactly where mechanism traces land. */
+async function walkRunDir(runDir: string, warnings: string[]): Promise<string[]> {
+  const files: string[] = []
+  const queue: string[] = [runDir]
+  while (queue.length > 0) {
+    const dir = queue.shift()!
+    const entries = await readdir(dir, { withFileTypes: true }).catch(() => [])
+    for (const entry of entries) {
+      const abs = join(dir, entry.name)
+      const rel = relative(runDir, abs)
+      if (entry.isDirectory()) {
+        if (rel === 'ws') {
+          queue.push(join(abs, '.loops'))
+          continue
+        }
+        queue.push(abs)
+      } else if (entry.isFile()) {
+        files.push(abs)
+        if (files.length >= MAX_FILES_PER_RUN_DIR) {
+          warnings.push(`${runDir}: file walk capped at ${MAX_FILES_PER_RUN_DIR} files`)
+          return files
+        }
+      }
+    }
+  }
+  return files
+}
+
+/** Run the predicate over the candidate's run dirs. Fail-closed: an unreadable
+ *  artifact contributes nothing (with a warning) — it can never count as
+ *  "fired". */
+export async function runActivationPredicate(
+  predicate: ActivationPredicate,
+  runDirs: readonly string[],
+): Promise<ActivationResult> {
+  const result: ActivationResult = { fired: false, evidence: [], checkedRunDirs: 0, checkedFiles: 0, warnings: [] }
+  for (const runDir of runDirs) {
+    result.checkedRunDirs += 1
+    if (predicate.kind === 'script') {
+      const res = await run('bash', ['-c', predicate.script!], {
+        cwd: runDir,
+        timeoutMs: SCRIPT_TIMEOUT_MS,
+        env: { ...process.env, RUN_DIR: runDir },
+      })
+      if (res.code === 0) {
+        result.fired = true
+        if (result.evidence.length < MAX_EVIDENCE) {
+          result.evidence.push(`${runDir}: script rc=0${res.stdout.trim() ? ` — ${res.stdout.trim().slice(0, 200)}` : ''}`)
+        }
+      } else if (res.timedOut) {
+        result.warnings.push(`${runDir}: script timed out after ${SCRIPT_TIMEOUT_MS}ms (counted as not-fired)`)
+      }
+      continue
+    }
+    const regex = new RegExp(predicate.pattern!)
+    const filters = (predicate.files ?? []).filter((f) => f.length > 0)
+    for (const file of await walkRunDir(runDir, result.warnings)) {
+      const rel = relative(runDir, file)
+      if (filters.length > 0 && !filters.some((f) => rel.includes(f))) continue
+      const info = await stat(file).catch(() => null)
+      if (info === null) continue
+      if (info.size > MAX_FILE_BYTES) {
+        result.warnings.push(`${rel}: skipped (${info.size} bytes > ${MAX_FILE_BYTES})`)
+        continue
+      }
+      result.checkedFiles += 1
+      const content = await readFile(file, 'utf8').catch(() => null)
+      if (content === null) continue
+      const lines = content.split('\n')
+      for (let i = 0; i < lines.length; i++) {
+        if (regex.test(lines[i]!)) {
+          result.fired = true
+          if (result.evidence.length < MAX_EVIDENCE) {
+            result.evidence.push(`${file}:${i + 1}: ${lines[i]!.trim().slice(0, 200)}`)
+          }
+          break // one match per file is enough evidence
+        }
+      }
+    }
+  }
+  return result
+}
+
+/** Read + parse the predicate committed at a candidate's loops commit. */
+export async function readCommittedPredicate(
+  loopsRepo: string,
+  commit: string,
+): Promise<{ raw: string; parsed: ParsedPredicate } | null> {
+  const show = await run('git', ['-C', loopsRepo, 'show', `${commit}:${ACTIVATION_PREDICATE_RELPATH}`])
+  if (show.code !== 0) return null
+  return { raw: show.stdout, parsed: parseActivationPredicate(show.stdout) }
+}
+
+/** The staircase row's activation record. */
+export interface ActivationRecord {
+  /** Whether a parseable predicate was present on the candidate commit. */
+  present: boolean
+  description: string | null
+  /** null = not evaluated (no predicate / gate disabled / baseline). */
+  fired: boolean | null
+  evidence: string[]
+  warnings: string[]
+}

package/src/swe-arena/activation.test.mts

@@ -0,0 +1,303 @@
+/**
+ * Gen-5 activation gate: predicate validation, execution over run artifacts
+ * (grep + script, ws/ skip with ws/.loops searched), the prefilter kill for a
+ * missing/invalid predicate, and the quarantine verdict path (an improved
+ * score with a never-fired mechanism is quarantined, not promoted).
+ */
+
+import { existsSync } from 'node:fs'
+import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import type { AnalystFinding } from '@tangle-network/agent-eval'
+import { afterEach, beforeEach, describe, expect, it } from 'vitest'
+import {
+  ACTIVATION_PREDICATE_RELPATH,
+  activationPredicateInstruction,
+  parseActivationPredicate,
+  runActivationPredicate,
+} from './activation.mts'
+import { decideVerdict } from './cell-evidence.mts'
+import { defaultRound4Config, parseStaircaseRow, STAIRCASE_SCHEMA, type OuterLoopConfig, type StaircaseRow } from './outer-loop.mts'
+import { fanOutLoopsGenerator, type ProposerSpec } from './proposer-fanout.mts'
+import { runOk } from './proc.ts'
+
+describe('parseActivationPredicate', () => {
+  it('accepts a valid grep predicate (with optional files filters)', () => {
+    const parsed = parseActivationPredicate(
+      JSON.stringify({
+        version: 'v1',
+        description: 'patchRiskWarnings emitted in at least one settle',
+        kind: 'grep',
+        pattern: 'patchRiskWarnings|patch-risk',
+        files: ['journal.jsonl', 'workers/'],
+      }),
+    )
+    expect(parsed.ok).toBe(true)
+  })
+
+  it('accepts a valid script predicate', () => {
+    const parsed = parseActivationPredicate(
+      JSON.stringify({ version: 'v1', description: 'x', kind: 'script', script: 'grep -rq X .' }),
+    )
+    expect(parsed.ok).toBe(true)
+  })
+
+  it.each([
+    ['not json', 'not valid JSON'],
+    ['[]', 'JSON object'],
+    [JSON.stringify({ version: 'v2', description: 'x', kind: 'grep', pattern: 'a' }), 'version'],
+    [JSON.stringify({ version: 'v1', description: '', kind: 'grep', pattern: 'a' }), 'description'],
+    [JSON.stringify({ version: 'v1', description: 'x', kind: 'grep' }), 'pattern'],
+    [JSON.stringify({ version: 'v1', description: 'x', kind: 'grep', pattern: '(' }), 'RegExp'],
+    [JSON.stringify({ version: 'v1', description: 'x', kind: 'grep', pattern: 'a', files: [1] }), 'files'],
+    [JSON.stringify({ version: 'v1', description: 'x', kind: 'script' }), 'script'],
+    [JSON.stringify({ version: 'v1', description: 'x', kind: 'sql' }), 'kind'],
+  ])('rejects %s', (raw, want) => {
+    const parsed = parseActivationPredicate(raw)
+    expect(parsed.ok).toBe(false)
+    if (!parsed.ok) expect(parsed.error).toContain(want)
+  })
+
+  it('the prompt instruction carries the template and the quarantine warning', () => {
+    const text = activationPredicateInstruction()
+    expect(text).toContain(ACTIVATION_PREDICATE_RELPATH)
+    expect(text).toContain('QUARANTINED')
+    expect(text).toContain('"kind": "grep"')
+  })
+})
+
+describe('runActivationPredicate', () => {
+  let runA: string
+  let runB: string
+
+  beforeEach(async () => {
+    runA = await mkdtemp(join(tmpdir(), 'act-a-'))
+    runB = await mkdtemp(join(tmpdir(), 'act-b-'))
+    await writeFile(join(runA, 'driver.log'), 'boot\nno mechanism here\n')
+    await writeFile(join(runB, 'driver.log'), 'boot\npatchRiskWarnings: 2 warnings emitted\n')
+  })
+
+  afterEach(async () => {
+    await rm(runA, { recursive: true, force: true })
+    await rm(runB, { recursive: true, force: true })
+  })
+
+  const grep = (pattern: string, files?: string[]) => ({
+    version: 'v1' as const,
+    description: 'd',
+    kind: 'grep' as const,
+    pattern,
+    ...(files ? { files } : {}),
+  })
+
+  it('fires with path:line evidence when the pattern appears in ANY run dir', async () => {
+    const res = await runActivationPredicate(grep('patchRiskWarnings'), [runA, runB])
+    expect(res.fired).toBe(true)
+    expect(res.checkedRunDirs).toBe(2)
+    expect(res.evidence[0]).toContain(join(runB, 'driver.log'))
+    expect(res.evidence[0]).toContain('patchRiskWarnings')
+  })
+
+  it('does not fire when the pattern never appears (fail-closed)', async () => {
+    const res = await runActivationPredicate(grep('neverEverPresent'), [runA, runB])
+    expect(res.fired).toBe(false)
+    expect(res.evidence).toEqual([])
+  })
+
+  it('honors files filters (relative-path substring)', async () => {
+    await writeFile(join(runB, 'other.txt'), 'patchRiskWarnings\n')
+    const onlyOther = await runActivationPredicate(grep('patchRiskWarnings', ['other.txt']), [runB])
+    expect(onlyOther.fired).toBe(true)
+    const onlyMissing = await runActivationPredicate(grep('patchRiskWarnings', ['nope.bin']), [runB])
+    expect(onlyMissing.fired).toBe(false)
+  })
+
+  it('skips the ws/ workspace subtree EXCEPT ws/.loops (supervisor artifacts)', async () => {
+    // Marker only inside ws/ (a checked-out repo) → must NOT count as fired.
+    await mkdir(join(runA, 'ws', 'src'), { recursive: true })
+    await writeFile(join(runA, 'ws', 'src', 'code.py'), 'patchRiskWarnings in the repo source\n')
+    const inWs = await runActivationPredicate(grep('patchRiskWarnings'), [runA])
+    expect(inWs.fired).toBe(false)
+    // Marker in ws/.loops → the supervisor's own artifacts, searched.
+    await mkdir(join(runA, 'ws', '.loops', 'supervisor', 's1'), { recursive: true })
+    await writeFile(join(runA, 'ws', '.loops', 'supervisor', 's1', 'journal.jsonl'), '{"k":"patchRiskWarnings"}\n')
+    const inLoops = await runActivationPredicate(grep('patchRiskWarnings'), [runA])
+    expect(inLoops.fired).toBe(true)
+  })
+
+  it('script kind fires on rc=0 in any run dir, not-fired otherwise', async () => {
+    const script = (s: string) => ({ version: 'v1' as const, description: 'd', kind: 'script' as const, script: s })
+    const hit = await runActivationPredicate(script('grep -q patchRiskWarnings driver.log'), [runA, runB])
+    expect(hit.fired).toBe(true)
+    expect(hit.evidence[0]).toContain(runB)
+    const miss = await runActivationPredicate(script('grep -q neverEverPresent driver.log'), [runA, runB])
+    expect(miss.fired).toBe(false)
+  })
+})
+
+describe('quarantine verdict path', () => {
+  const base = {
+    violations: [],
+    coverageComplete: true,
+    resolvedCount: 3,
+    parentResolvedCount: 1,
+    costRatio: 1.0,
+    costGuardRatio: 1.2,
+  }
+
+  it('quarantines a candidate whose mechanism never fired EVEN when its score improved', () => {
+    expect(decideVerdict({ ...base, activationFired: false })).toBe('quarantined-inactive')
+  })
+
+  it('accepts an improved candidate whose mechanism fired; gate-not-applicable behaves as before', () => {
+    expect(decideVerdict({ ...base, activationFired: true })).toBe('accepted')
+    expect(decideVerdict({ ...base, activationFired: null })).toBe('accepted')
+    expect(decideVerdict(base)).toBe('accepted')
+  })
+
+  it('out-of-space still wins over quarantine; quarantine wins over no-gain/cost/coverage', () => {
+    expect(decideVerdict({ ...base, violations: ['judge.py'], activationFired: false })).toBe('rejected-out-of-space')
+    expect(decideVerdict({ ...base, coverageComplete: false, activationFired: false })).toBe('quarantined-inactive')
+    expect(decideVerdict({ ...base, resolvedCount: 1, activationFired: false })).toBe('quarantined-inactive')
+  })
+
+  it('parseStaircaseRow accepts a quarantined row with gen-5 split + activation fields', () => {
+    const row: StaircaseRow = {
+      schema: STAIRCASE_SCHEMA,
+      round: 4,
+      generation: 0,
+      runId: 'r4-x',
+      at: new Date(0).toISOString(),
+      candidate: 'hash',
+      candidateCommit: 'c'.repeat(40),
+      parent: 'baseline',
+      parentResolvedCount: 1,
+      changedFiles: [],
+      changeSpaceViolations: [],
+      perInstance: [],
+      resolvedCount: 3,
+      coverageComplete: true,
+      wallS: 10,
+      baselineWallS: 10,
+      costRatio: 1,
+      costGuardRatio: 1.2,
+      internallyPromoted: true,
+      verdict: 'quarantined-inactive',
+      holdout: 'operator-approval-required',
+      armProvenance: null,
+      diffPath: null,
+      diffSha256: null,
+      split: {
+        publicInstances: ['a', 'b'],
+        privateInstances: ['c'],
+        publicResolvedCount: 2,
+        privateResolvedCount: 1,
+      },
+      activation: { present: true, description: 'd', fired: false, evidence: [], warnings: [] },
+    }
+    expect(parseStaircaseRow(JSON.stringify(row))).toEqual(row)
+  })
+})
+
+// ---------------------------------------------------------------------------
+// Prefilter enforcement — a candidate without a parseable predicate is killed
+// before any evaluation spend.
+// ---------------------------------------------------------------------------
+
+describe('activation-predicate prefilter', () => {
+  let loopsRepo: string
+  let outDir: string
+  let driverWt: string
+
+  beforeEach(async () => {
+    loopsRepo = await mkdtemp(join(tmpdir(), 'act-repo-'))
+    outDir = await mkdtemp(join(tmpdir(), 'act-out-'))
+    await runOk('git', ['init', '-q', '-b', 'main', loopsRepo])
+    await runOk('git', ['-C', loopsRepo, 'config', 'user.email', 't@t.dev'])
+    await runOk('git', ['-C', loopsRepo, 'config', 'user.name', 'T'])
+    await writeFile(join(loopsRepo, 'src.ts'), 'base\n')
+    await runOk('git', ['-C', loopsRepo, 'add', '-A'])
+    await runOk('git', ['-C', loopsRepo, 'commit', '-q', '-m', 'init'])
+    driverWt = join(outDir, 'driver-wt')
+    await runOk('git', ['-C', loopsRepo, 'worktree', 'add', '--detach', driverWt, 'HEAD'])
+  })
+
+  afterEach(async () => {
+    await rm(outDir, { recursive: true, force: true })
+    await rm(loopsRepo, { recursive: true, force: true })
+  })
+
+  const config = (proposers: ProposerSpec[]): OuterLoopConfig => ({
+    ...defaultRound4Config(),
+    loopsRepo,
+    outDir,
+    populationSize: proposers.length,
+    proposers,
+    activationGate: true,
+  })
+
+  const generatorArgs = (candidateIndex: number) => ({
+    worktreePath: driverWt,
+    report: undefined,
+    findings: [] as AnalystFinding[],
+    maxShots: 1,
+    signal: new AbortController().signal,
+    generation: 0,
+    candidateIndex,
+  })
+
+  it('kills a candidate without .improve/activation.json (stage activation-predicate) and passes one WITH it', async () => {
+    const proposers: ProposerSpec[] = [
+      { name: 'with-predicate', harness: 'claude' },
+      { name: 'without-predicate', harness: 'claude' },
+      { name: 'invalid-predicate', harness: 'claude' },
+    ]
+    const gen = fanOutLoopsGenerator(config(proposers), {
+      author: async (proposer, args) => {
+        await mkdir(join(args.worktreePath, 'extensions', 'pi'), { recursive: true })
+        await writeFile(join(args.worktreePath, 'extensions', 'pi', `${proposer.name}.ts`), 'x\n')
+        if (proposer.name === 'with-predicate') {
+          await mkdir(join(args.worktreePath, '.improve'), { recursive: true })
+          await writeFile(
+            join(args.worktreePath, ACTIVATION_PREDICATE_RELPATH),
+            JSON.stringify({ version: 'v1', description: 'd', kind: 'grep', pattern: 'x' }),
+          )
+        } else if (proposer.name === 'invalid-predicate') {
+          await mkdir(join(args.worktreePath, '.improve'), { recursive: true })
+          await writeFile(join(args.worktreePath, ACTIVATION_PREDICATE_RELPATH), '{"version":"v1"}')
+        }
+        return { applied: true, summary: `${proposer.name} edit` }
+      },
+    })
+
+    const survivor = await gen.generate(generatorArgs(0))
+    const missing = await gen.generate(generatorArgs(1))
+    const invalid = await gen.generate(generatorArgs(2))
+
+    expect(survivor.applied).toBe(true)
+    expect(missing.applied).toBe(false)
+    expect(invalid.applied).toBe(false)
+    const kills = gen.drainPrefilterKills()
+    expect(kills).toHaveLength(2)
+    expect(kills.find((k) => k.proposer === 'without-predicate')).toMatchObject({ stage: 'activation-predicate' })
+    expect(kills.find((k) => k.proposer === 'without-predicate')!.reason).toContain('missing')
+    expect(kills.find((k) => k.proposer === 'invalid-predicate')!.reason).toContain('invalid')
+    // The survivor's predicate landed on the driver worktree with the patch.
+    expect(existsSync(join(driverWt, ACTIVATION_PREDICATE_RELPATH))).toBe(true)
+  })
+
+  it('does not require a predicate when the gate is off (gen-4 behavior unchanged)', async () => {
+    const cfg = config([{ name: 'legacy', harness: 'claude' }])
+    cfg.activationGate = false
+    const gen = fanOutLoopsGenerator(cfg, {
+      author: async (_p, args) => {
+        await mkdir(join(args.worktreePath, 'extensions', 'pi'), { recursive: true })
+        await writeFile(join(args.worktreePath, 'extensions', 'pi', 'l.ts'), 'x\n')
+        return { applied: true, summary: 'edit' }
+      },
+    })
+    expect((await gen.generate(generatorArgs(0))).applied).toBe(true)
+    expect(gen.drainPrefilterKills()).toEqual([])
+  })
+})