@vltpkg/query 1.5.0 → 1.6.0

package/skills/dss-query/evals/grade.mjs

@@ -1,481 +0,0 @@
-/**
- * Deterministic grader for dss-query skill evals.
- * Run from src/query (so @vltpkg/* workspace deps resolve):
- *   node skills/dss-query/evals/grade.mjs skills/dss-query-workspace/iteration-1
- *
- * Writes grading.json ({expectations: [{text, passed, evidence}]}) into each
- * <eval>/<variant>/ directory.
- */
-import {
-  readFileSync,
-  writeFileSync,
-  readdirSync,
-  existsSync,
-} from 'node:fs'
-import { join } from 'node:path'
-import { parse } from '@vltpkg/dss-parser'
-import { Query } from '../../../src/index.ts'
-import { getSimpleGraph } from '../../../test/fixtures/graph.ts'
-
-const iterDir = process.argv[2]
-if (!iterDir) throw new Error('usage: node grade.mjs <iteration-dir>')
-
-// ---- selector extraction -------------------------------------------------
-const extractSelectors = text => {
-  const out = new Set()
-  // vlt query '<sel>' / "<sel>" anywhere in the text
-  for (const m of text.matchAll(/vlt query '([^']+)'/g)) out.add(m[1])
-  for (const m of text.matchAll(/vlt query "([^"]+)"/g)) out.add(m[1])
-  // bare selectors on their own line inside fenced code blocks
-  for (const block of text.matchAll(/```[a-z]*\n([\s\S]*?)```/g)) {
-    for (const line of block[1].split('\n')) {
-      const t = line.trim()
-      // '# ...' is a comment, not an ID selector (real IDs: '#react', no space)
-      if (/^# /.test(t)) continue
-      if (/^[:#[*]/.test(t) && !t.includes('<')) out.add(t)
-    }
-  }
-  return [...out].filter(s => !s.includes('<'))
-}
-
-// ---- execution (offline only) ---------------------------------------------
-// :outdated fetches registry packuments; security selectors need the archive.
-const NO_EXEC = /:outdated|:missing/
-const skipExec = s => Query.hasSecuritySelectors(s) || NO_EXEC.test(s)
-// split top-level commas so loose mode can't mask invalid segments
-const topLevelSplit = s => {
-  const parts = []
-  let depth = 0,
-    cur = ''
-  for (const ch of s) {
-    if (ch === '(' || ch === '[') depth++
-    else if (ch === ')' || ch === ']') depth--
-    if (ch === ',' && depth === 0) {
-      parts.push(cur)
-      cur = ''
-    } else cur += ch
-  }
-  parts.push(cur)
-  return parts.map(p => p.trim()).filter(Boolean)
-}
-const mockSignal = { throwIfAborted: () => {} }
-const makeQuery = () => {
-  const graph = getSimpleGraph()
-  return new Query({
-    nodes: new Set(graph.nodes.values()),
-    edges: graph.edges,
-    importers: graph.importers,
-    securityArchive: undefined,
-  })
-}
-
-// parse() rejects invalid DSS (grammar + structure), not just bad syntax
-const parseAll = selectors => {
-  const failures = []
-  for (const s of selectors) {
-    try {
-      parse(s)
-    } catch (err) {
-      failures.push(`${s} → ${err.message}`)
-    }
-  }
-  return failures
-}
-
-const execAll = async selectors => {
-  const failures = []
-  const skipped = []
-  for (const s of selectors) {
-    if (skipExec(s)) {
-      skipped.push(s)
-      continue
-    }
-    for (const seg of topLevelSplit(s)) {
-      try {
-        await makeQuery().search(seg, { signal: mockSignal })
-      } catch (err) {
-        failures.push(`${seg} → ${err.message}`)
-      }
-    }
-  }
-  return { failures, skipped }
-}
-
-// ---- per-eval assertion logic ----------------------------------------------
-const first = (arr, re) => arr.find(s => re.test(s))
-const preCodeText = text =>
-  text.slice(0, Math.max(text.indexOf('```'), 0)).trim()
-
-const graders = {
-  0: async (text, sels) => {
-    const hit = first(sels, /:root\s*>\s*\*?:outdated/)
-    const parseFails = parseAll(sels)
-    const { failures: execFails, skipped } = await execAll(sels)
-    const pre = preCodeText(text)
-    return [
-      {
-        text: 'proposes-root-outdated-selector',
-        passed: !!hit,
-        evidence: hit ?? `selectors: ${sels.join(' | ') || '(none)'}`,
-      },
-      {
-        text: 'all-selectors-parse',
-        passed: parseFails.length === 0,
-        evidence:
-          parseFails.join('; ') || `${sels.length} selectors parsed`,
-      },
-      {
-        text: 'selector-executes-on-fixture-graph',
-        passed: execFails.length === 0,
-        evidence:
-          execFails.join('; ') ||
-          `executed ${sels.length - skipped.length}, skipped (network-dependent): ${skipped.length}`,
-      },
-      {
-        text: 'query-first-style',
-        passed: pre.length < 150,
-        evidence:
-          pre.length < 150 ?
-            `only ${pre.length} chars before first code block`
-          : `${pre.length} chars of preamble before first code block: "${pre.slice(0, 80)}..."`,
-      },
-      {
-        text: 'mentions-view-flag',
-        passed: /--view/.test(text),
-        evidence:
-          /--view/.test(text) ?
-            `matched "${text.match(/--view\S*/)[0]}"`
-          : 'no --view mention',
-      },
-      {
-        text: 'offers-related-queries',
-        passed: sels.length >= 2,
-        evidence: `${sels.length} distinct selectors offered`,
-      },
-    ]
-  },
-  1: async (text, sels) => {
-    const attr = first(sels, /\[license\^?=\s*['"]?MIT/i)
-    const badPseudo = /:license\(\s*['"]?mit/i.test(text)
-    const anchored = first(sels, /:workspace\s*>\s*:prod/)
-    const parseFails = parseAll(sels)
-    return [
-      {
-        text: 'uses-license-attribute-form',
-        passed: !!attr,
-        evidence:
-          attr ?? `selectors: ${sels.join(' | ') || '(none)'}`,
-      },
-      {
-        text: 'avoids-invalid-license-pseudo',
-        passed: !badPseudo,
-        evidence:
-          badPseudo ?
-            'answer proposes :license(mit), which throws "Expected a valid license kind"'
-          : 'no :license(mit) present',
-      },
-      {
-        text: 'workspace-prod-anchored',
-        passed: !!anchored,
-        evidence:
-          anchored ?? `selectors: ${sels.join(' | ') || '(none)'}`,
-      },
-      {
-        text: 'all-selectors-parse',
-        passed: parseFails.length === 0,
-        evidence:
-          parseFails.join('; ') || `${sels.length} selectors parsed`,
-      },
-      {
-        text: 'offers-related-queries',
-        passed: sels.length >= 2,
-        evidence: `${sels.length} distinct selectors offered`,
-      },
-    ]
-  },
-  2: async (text, sels) => {
-    const inv = first(
-      sels,
-      /:has\(\s*>\s*(#react|\[name=react\])\s*\)/,
-    )
-    const correction =
-      /(#react\s*>\s*\*|wrong way|opposite direction|points? the wrong|selects? react'?s (own )?dependenc)/i.test(
-        text,
-      ) && /dependenc/i.test(text)
-    // the recommended (first extracted) selector must not be the wrong-direction one
-    const firstSel = sels[0] ?? ''
-    const wrongAsAnswer = /^#react\s*>\s*\*$/.test(firstSel)
-    const parseFails = parseAll(sels)
-    const evidenceLine =
-      text
-        .split('\n')
-        .find(l => /wrong|opposite|dependenc/i.test(l)) ?? ''
-    return [
-      {
-        text: 'uses-has-inversion',
-        passed: !!inv,
-        evidence: inv ?? `selectors: ${sels.join(' | ') || '(none)'}`,
-      },
-      {
-        text: 'corrects-direction-mistake',
-        passed: correction,
-        evidence:
-          evidenceLine.trim().slice(0, 160) ||
-          'no direction-correction language found',
-      },
-      {
-        text: 'does-not-present-wrong-selector-as-answer',
-        passed: !wrongAsAnswer,
-        evidence: `first recommended selector: ${firstSel || '(none)'}`,
-      },
-      {
-        text: 'all-selectors-parse',
-        passed: parseFails.length === 0,
-        evidence:
-          parseFails.join('; ') || `${sels.length} selectors parsed`,
-      },
-      {
-        text: 'explains-combinator-rationale',
-        passed:
-          /point.{0,40}dependenc|dependent\s*(→|->|to)\s*dependenc|parent.{0,12}child/i.test(
-            text,
-          ),
-        evidence: (
-          text.split('\n').find(l => /point|parent|→|->/.test(l)) ??
-          'no general direction rule stated'
-        )
-          .trim()
-          .slice(0, 160),
-      },
-    ]
-  },
-  3: async text => {
-    const checks = [
-      ['covers-workspace-anchor', /workspace/i],
-      ['covers-direct-prod-segment', null], // special: needs both
-      ['covers-cve-segment', /cve/i],
-      ['mentions-security-data-source', /socket|network/i],
-    ]
-    return checks.map(([name, re]) => {
-      if (name === 'covers-direct-prod-segment') {
-        const ok = /direct/i.test(text) && /prod/i.test(text)
-        return {
-          text: name,
-          passed: ok,
-          evidence:
-            ok ?
-              'mentions both "direct" and "prod"'
-            : `direct:${/direct/i.test(text)} prod:${/prod/i.test(text)}`,
-        }
-      }
-      const m = text.match(re)
-      return {
-        text: name,
-        passed: !!m,
-        evidence: m ? `matched "${m[0]}"` : `no match for ${re}`,
-      }
-    })
-  },
-  4: async (text, sels) => {
-    const threat =
-      /:malware/.test(text) &&
-      /:squat|:obfuscated|:suspicious/.test(text)
-    const parseFails = parseAll(sels)
-    return [
-      {
-        text: 'uses-threat-selectors',
-        passed: threat,
-        evidence:
-          threat ?
-            `selectors: ${sels.join(' | ')}`
-          : `malware:${/:malware/.test(text)} squat/obf/susp:${/:squat|:obfuscated|:suspicious/.test(text)}`,
-      },
-      {
-        text: 'all-selectors-parse',
-        passed: parseFails.length === 0,
-        evidence:
-          parseFails.join('; ') || `${sels.length} selectors parsed`,
-      },
-      {
-        text: 'mentions-security-data-source',
-        passed: /socket|network/i.test(text),
-        evidence:
-          text.match(/socket|network/i)?.[0] ??
-          'no Socket/network mention',
-      },
-      {
-        text: 'offers-related-queries',
-        passed: sels.length >= 2,
-        evidence: `${sels.length} distinct selectors offered`,
-      },
-    ]
-  },
-  5: async (text, sels) => {
-    const cap =
-      /:dev[^\s,']*:(shell|network)|:dev:is\([^)]*:(shell|network)/.test(
-        text,
-      )
-    const both = /:shell/.test(text) && /:network/.test(text)
-    const parseFails = parseAll(sels)
-    return [
-      {
-        text: 'uses-capability-selectors',
-        passed: cap,
-        evidence:
-          cap ?
-            `selectors: ${sels.join(' | ')}`
-          : `selectors: ${sels.join(' | ') || '(none)'}`,
-      },
-      {
-        text: 'covers-both-capabilities',
-        passed: both,
-        evidence: `shell:${/:shell/.test(text)} network:${/:network/.test(text)}`,
-      },
-      {
-        text: 'all-selectors-parse',
-        passed: parseFails.length === 0,
-        evidence:
-          parseFails.join('; ') || `${sels.length} selectors parsed`,
-      },
-      {
-        text: 'explains-capability-meaning',
-        passed: /socket|static|capab|can\s/i.test(text),
-        evidence:
-          text.match(/socket|static|capab|can\s\S*/i)?.[0] ??
-          'no capability explanation',
-      },
-    ]
-  },
-  6: async (text, sels) => {
-    const sev =
-      first(sels, /:sev(erity)?\(/) ??
-      (/:sev(erity)?\(/.test(text) ?
-        text.match(/:sev(erity)?\([^)]*\)/)[0]
-      : undefined)
-    const anchored = /:workspace\s*>\s*:prod/.test(text)
-    const parseFails = parseAll(sels)
-    return [
-      {
-        text: 'uses-severity-filter',
-        passed: !!sev,
-        evidence: sev ?? `selectors: ${sels.join(' | ') || '(none)'}`,
-      },
-      {
-        text: 'anchors-workspace-prod',
-        passed: anchored,
-        evidence:
-          anchored ?
-            text.match(/:workspace\s*>\s*:prod\S*/)[0]
-          : `selectors: ${sels.join(' | ') || '(none)'}`,
-      },
-      {
-        text: 'all-selectors-parse',
-        passed: parseFails.length === 0,
-        evidence:
-          parseFails.join('; ') || `${sels.length} selectors parsed`,
-      },
-      {
-        text: 'mentions-security-data-source',
-        passed: /socket|network/i.test(text),
-        evidence:
-          text.match(/socket|network/i)?.[0] ??
-          'no Socket/network mention',
-      },
-    ]
-  },
-}
-
-// ---- round-3 style assertions (Examples tail, docs deep links, concision) ---
-const examplesHeading = text => ({
-  text: 'uses-examples-heading',
-  passed: /Examples:/.test(text) && !/Related:/.test(text),
-  evidence: `Examples: ${/Examples:/.test(text)}, Related: ${/Related:/.test(text)}`,
-})
-const docsLink = re => text => ({
-  text: 'ends-with-docs-link',
-  passed: re.test(text),
-  evidence: text.match(re)?.[0] ?? `no link matching ${re}`,
-})
-const conciseAnswer = max => text => {
-  const lines = text.split('\n').filter(l => l.trim()).length
-  return {
-    text: 'concise-answer',
-    passed: lines <= max,
-    evidence: `${lines} non-empty lines (max ${max})`,
-  }
-}
-const SELECTORS_DOC = /docs\.vlt\.io\/cli\/selectors/
-const round3Extras = {
-  0: [examplesHeading, docsLink(SELECTORS_DOC)],
-  1: [
-    examplesHeading,
-    // license query: either the structural selectors page or the
-    // license-compliance section is a correct deep link
-    docsLink(
-      /docs\.vlt\.io\/cli\/(selectors|security#license-compliance)/,
-    ),
-  ],
-  2: [docsLink(SELECTORS_DOC)],
-  3: [
-    docsLink(/docs\.vlt\.io\/cli\/security#vulnerability-detection/),
-  ],
-  4: [docsLink(/docs\.vlt\.io\/cli\/security#malware-detection/)],
-  5: [
-    conciseAnswer(22),
-    docsLink(
-      /docs\.vlt\.io\/cli\/security#behavioral-security-risks/,
-    ),
-  ],
-  6: [
-    examplesHeading,
-    docsLink(/docs\.vlt\.io\/cli\/security#vulnerability-detection/),
-  ],
-}
-
-// ---- run --------------------------------------------------------------------
-for (const evalDir of readdirSync(iterDir).filter(d =>
-  d.startsWith('eval-'),
-)) {
-  const id = Number(evalDir.match(/^eval-(\d+)/)[1])
-  const variants = readdirSync(join(iterDir, evalDir), {
-    withFileTypes: true,
-  })
-    .filter(d => d.isDirectory())
-    .map(d => d.name)
-  for (const variant of variants) {
-    const answerPath = join(
-      iterDir,
-      evalDir,
-      variant,
-      'run-1',
-      'outputs',
-      'answer.md',
-    )
-    if (!existsSync(answerPath)) continue
-    const text = readFileSync(answerPath, 'utf8')
-    const sels = extractSelectors(text)
-    const expectations = [
-      ...(await graders[id](text, sels)),
-      ...(round3Extras[id] ?? []).map(fn => fn(text)),
-    ]
-    const passed = expectations.filter(e => e.passed).length
-    writeFileSync(
-      join(iterDir, evalDir, variant, 'run-1', 'grading.json'),
-      JSON.stringify(
-        {
-          summary: {
-            total: expectations.length,
-            passed,
-            pass_rate: passed / expectations.length,
-          },
-          expectations,
-        },
-        null,
-        2,
-      ) + '\n',
-    )
-    console.log(
-      `${evalDir}/${variant}: ${passed}/${expectations.length}`,
-    )
-  }
-}

package/src/attribute.ts

@@ -1,190 +0,0 @@
-import { asAttributeNode } from '@vltpkg/dss-parser'
-import { error } from '@vltpkg/error-cause'
-import { queryError } from './error.ts'
-import { removeDanglingEdges } from './pseudo/helpers.ts'
-import type { AttributeOperatorName } from '@vltpkg/dss-parser'
-import type { NodeLike, JSONField, Manifest } from '@vltpkg/types'
-import type { ParserState } from './types.ts'
-
-export type ComparatorFn = (attr: string, value?: string) => boolean
-
-// JSONField has a mapped type constituent that would coerce to [object Object]
-// when stringified, which is what we want in this case.
-// eslint-disable-next-line @typescript-eslint/no-base-to-string
-const jsonFieldToString = (v: JSONField) => String(v)
-
-/**
- * Retrieve the {@link Manifest} values found at the given `properties`
- * location for a given {@link Node}.
- */
-export const getManifestPropertyValues = (
-  node: NodeLike,
-  properties: string[],
-  attribute: string,
-): string[] | undefined => {
-  if (!node.manifest) return
-
-  const traverse = new Set<JSONField>([node.manifest])
-  const props = new Set<JSONField>()
-  for (const key of properties) {
-    for (const prop of traverse) {
-      /* c8 ignore start - should be impossible */
-      if (!prop) {
-        throw error('failed to find nested property in :attr', {
-          found: properties,
-        })
-      }
-      /* c8 ignore stop */
-
-      // expand the result list to include nested array values
-      if (Array.isArray(prop)) {
-        for (const p of prop) {
-          traverse.add(p)
-        }
-        continue
-      }
-
-      // guard for inspecting keys of objects next
-      if (
-        typeof prop === 'string' ||
-        typeof prop === 'number' ||
-        typeof prop === 'boolean'
-      ) {
-        continue
-      }
-
-      // assign next value when found
-      if (key in prop) {
-        const nextValue = prop[key]
-        if (nextValue) {
-          if (key === attribute) {
-            props.add(nextValue)
-          } else {
-            traverse.delete(prop)
-            traverse.add(nextValue)
-          }
-        }
-      }
-    }
-  }
-  // if no value was found after trying a given key
-  // then there's nothing to be collected
-  if (!props.size) return
-
-  // expand the result to include array values
-  const collect = new Set<string>()
-  for (const prop of props) {
-    if (Array.isArray(prop)) {
-      for (const p of prop) {
-        collect.add(p ? jsonFieldToString(p) : '')
-      }
-    } else {
-      collect.add(jsonFieldToString(prop))
-    }
-  }
-
-  return [...collect]
-}
-
-// decorator style of function that will filter `ParserState` results
-// based on a provided `comparator` function
-export const filterAttributes = (
-  state: ParserState,
-  comparator: ComparatorFn | undefined,
-  value: string,
-  propertyName: string,
-  insensitive: boolean,
-  prefixProperties: string[] = [],
-): ParserState => {
-  const check = (attr: JSONField) =>
-    comparator?.(
-      insensitive ?
-        jsonFieldToString(attr).toLowerCase()
-      : jsonFieldToString(attr),
-      insensitive ? value.toLowerCase() : value,
-    )
-  const deleteNode = (node: NodeLike) => {
-    for (const edge of node.edgesIn) {
-      state.partial.edges.delete(edge)
-    }
-    state.partial.nodes.delete(node)
-  }
-
-  for (const node of state.partial.nodes) {
-    const prefixes =
-      prefixProperties.length ? prefixProperties : [propertyName]
-    const attrs = getManifestPropertyValues(
-      node,
-      prefixes,
-      propertyName,
-    )
-
-    // if no attribute value was found, that means the attribute won't match
-    if (!attrs?.length) {
-      deleteNode(node)
-      continue
-    }
-
-    // if the node attribute value won't match, then remove the node
-    if (comparator && !attrs.some(check)) {
-      deleteNode(node)
-    }
-  }
-
-  removeDanglingEdges(state)
-  return state
-}
-
-// ref: https://developer.mozilla.org/en-US/docs/Web/CSS/Attribute_selectors
-const attributeSelectors = {
-  '=': (attr: string, value = '') => attr === value,
-  '^=': (attr: string, value = '') => attr.startsWith(value),
-  '$=': (attr: string, value = '') => attr.endsWith(value),
-  '~=': (attr: string, value = '') =>
-    new Set<string>(attr.match(/\w+/g)).has(value),
-  '*=': (attr: string, value = '') => attr.includes(value),
-  '|=': (attr: string, value = '') =>
-    attr === value || attr.startsWith(`${value}-`),
-  undefined: (attr: string) => !!attr,
-} satisfies Record<AttributeOperatorName | 'undefined', ComparatorFn>
-export const attributeSelectorsMap = new Map<string, ComparatorFn>(
-  Object.entries(attributeSelectors),
-)
-
-/**
- * Parse attributes selectors, e.g: `[name]`, `[name=value]`, etc
- */
-export const attribute = async (
-  state: ParserState,
-): Promise<ParserState> => {
-  await state.cancellable()
-
-  const curr = asAttributeNode(state.current)
-  const operatorFn = attributeSelectorsMap.get(String(curr.operator))
-  if (!operatorFn) {
-    if (state.loose) {
-      return state
-    }
-
-    throw queryError(
-      `Unsupported attribute operator: ${curr.operator}`,
-      state.current,
-      { validOptions: [...attributeSelectorsMap.keys()] },
-    )
-  }
-
-  const value = curr.value || ''
-  const propertyName = curr.attribute
-  const insensitive = !!curr.insensitive
-
-  // Increment the commonCounter for specificity
-  state.specificity.commonCounter += 1
-
-  return filterAttributes(
-    state,
-    operatorFn,
-    value,
-    propertyName,
-    insensitive,
-  )
-}