omakit 0.4.1 → 0.4.3

package/tools/inspect/functions.mjs

@@ -0,0 +1,129 @@
+// Function sites: every named function, QML handler, shell function and
+// Python def, with its length in lines, its deepest nesting and its branch
+// count. Size is the one thing a person asks about a tree that regular
+// expressions can answer without a parser: where does the logic pile up.
+// The numbers are counts over the text; the threshold they are compared
+// with is measured over listed trees (M12), never chosen.
+
+import { blankComments, closingBracket, lineOf } from "./text.mjs"
+
+const JS_BRANCH = /\b(?:if|else if|for|while|do|switch|case|catch)\b|&&|\|\||\?[^.:]/g
+const SHELL_OPEN = /^\s*(?:if|for|while|until|case|select)\b/
+const SHELL_CLOSE = /^\s*(?:fi|done|esac)\b/
+const SHELL_BRANCH = /\b(?:if|elif|for|while|until|case)\b|\|\||&&|^\s*[^)]*\)\s*(?!\s*$)/
+const PY_BRANCH = /^\s*(?:if|elif|for|while|except|with)\b|\band\b|\bor\b/
+
+/** Deepest brace nesting inside a body, relative to the body itself. */
+function braceDepth(body) {
+  let depth = 0
+  let deepest = 0
+  let quote = null
+  for (let i = 0; i < body.length; i += 1) {
+    const ch = body[i]
+    if (quote) {
+      if (ch === "\\") i += 1
+      else if (ch === quote) quote = null
+      continue
+    }
+    if (ch === '"' || ch === "'" || ch === "`") quote = ch
+    else if (ch === "{") {
+      depth += 1
+      if (depth > deepest) deepest = depth
+    } else if (ch === "}") depth -= 1
+  }
+  return deepest
+}
+
+function jsFunctions(file) {
+  const text = blankComments(file.text)
+  const rows = []
+  // `function name(` and, in QML, `onSomething: {` handlers with a body block.
+  const pattern = /(?<![\w.])function\s+([A-Za-z_$][\w$]*)\s*\(|(?<![\w.])(on[A-Z]\w*)\s*:\s*(?=\{)/g
+  for (const match of text.matchAll(pattern)) {
+    const open = text.indexOf("{", match.index + match[0].length - (match[2] ? 0 : 0))
+    if (open < 0) continue
+    const end = closingBracket(text, open)
+    if (end < 0) continue
+    const body = text.slice(open + 1, end)
+    const start = lineOf(text, match.index)
+    rows.push({
+      file: file.path,
+      line: start,
+      name: match[1] || match[2],
+      kind: match[1] ? "function" : "handler",
+      lines: lineOf(text, end) - start + 1,
+      depth: braceDepth(body),
+      branches: (body.match(JS_BRANCH) || []).length,
+    })
+  }
+  return rows
+}
+
+function shellFunctions(file) {
+  const lines = file.text.split("\n")
+  const rows = []
+  for (let index = 0; index < lines.length; index += 1) {
+    const head = lines[index].match(/^\s*(?:function\s+)?([A-Za-z_][\w-]*)\s*\(\)\s*\{?\s*$|^\s*function\s+([A-Za-z_][\w-]*)\s*\{?\s*$/)
+    if (!head) continue
+    const name = head[1] || head[2]
+    // The body runs to the line that is only `}` at the function's own indent.
+    const indent = lines[index].match(/^\s*/)[0]
+    let end = index
+    let depth = 0
+    let deepest = 0
+    let branches = 0
+    for (let at = index + 1; at < lines.length; at += 1) {
+      const line = lines[at]
+      if (line.trim() === "}" && line.startsWith(indent) && line.match(/^\s*/)[0].length === indent.length) {
+        end = at
+        break
+      }
+      if (SHELL_OPEN.test(line)) {
+        depth += 1
+        if (depth > deepest) deepest = depth
+      }
+      if (SHELL_CLOSE.test(line)) depth -= 1
+      if (SHELL_BRANCH.test(line.split("#")[0])) branches += 1
+      end = at
+    }
+    rows.push({ file: file.path, line: index + 1, name, kind: "function", lines: end - index + 1, depth: deepest, branches })
+    index = end
+  }
+  return rows
+}
+
+function pythonFunctions(file) {
+  const lines = file.text.split("\n")
+  const rows = []
+  for (let index = 0; index < lines.length; index += 1) {
+    const head = lines[index].match(/^(\s*)(?:async\s+)?def\s+([A-Za-z_]\w*)\s*\(/)
+    if (!head) continue
+    const base = head[1].length
+    let end = index
+    let deepest = 0
+    let branches = 0
+    for (let at = index + 1; at < lines.length; at += 1) {
+      const line = lines[at]
+      if (!line.trim()) continue
+      const indent = line.match(/^\s*/)[0].length
+      if (indent <= base) break
+      const level = Math.floor((indent - base) / 4)
+      if (level > deepest) deepest = level
+      if (PY_BRANCH.test(line)) branches += 1
+      end = at
+    }
+    rows.push({ file: file.path, line: index + 1, name: head[2], kind: "function", lines: end - index + 1, depth: deepest, branches })
+  }
+  return rows
+}
+
+/**
+ * @param {{ path: string, kind: string, text: string }} file
+ * @returns {Array<{ file, line, name, kind, lines, depth, branches }>}
+ */
+export function extractFunctions(file) {
+  if (file.kind === "qml" || file.kind === "js") return jsFunctions(file)
+  if (file.kind === "shell") return shellFunctions(file)
+  if (file.kind === "python") return pythonFunctions(file)
+  return []
+}

package/tools/inspect/hosts.mjs

@@ -0,0 +1,141 @@
+// Host sites: every `http://` or `https://` literal in code or argv, with
+// the host and scheme it names, the tool the literal reaches (the argv it
+// sits in, or the call on its line), and the timeout and size-cap flags
+// that argv shows. A host built from a variable is not a host: a literal
+// whose host part holds `${...}` or stops at `://` is recorded as not
+// resolvable and never guessed.
+
+import { basename, blankComments, lineOf, shellSegments, shellWords } from "./text.mjs"
+import { toolOf } from "./processes.mjs"
+
+const URL = /https?:\/\/[^\s"'`)\]}>,;]*/g
+const TIMEOUT_FLAGS = ["--max-time", "-m", "--connect-timeout", "--timeout", "-T"]
+const SIZE_FLAGS = ["--max-filesize", "--quota", "-Q"]
+
+/** Loopback, link-local and RFC 1918 literals, and the names that resolve there. */
+export function privateAddress(host) {
+  const h = String(host).toLowerCase().replace(/^\[|\]$/g, "")
+  if (["localhost", "0.0.0.0", "::1", "::"].includes(h)) return true
+  const v4 = h.match(/^(\d+)\.(\d+)\.(\d+)\.(\d+)$/)
+  if (!v4) return /^f[cd][0-9a-f]{2}:|^fe80:/.test(h)
+  const [a, b] = [Number(v4[1]), Number(v4[2])]
+  return a === 10 || a === 127 || (a === 192 && b === 168) || (a === 172 && b >= 16 && b <= 31) || (a === 169 && b === 254) || (a === 100 && b >= 64 && b <= 127)
+}
+
+/** The host of a URL literal, or null when the text shows an expression where the host would be. */
+export function hostOf(url) {
+  const rest = url.replace(/^https?:\/\//, "")
+  const authority = rest.split(/[/?#]/)[0]
+  if (!authority || /\$\{|\$\(|["'`+]/.test(authority)) return null
+  const host = authority.includes("@") ? authority.slice(authority.lastIndexOf("@") + 1) : authority
+  const bare = host.startsWith("[") ? host.slice(0, host.indexOf("]") + 1) : host.split(":")[0]
+  return bare && /^[\w.\-[\]:]+$/.test(bare) ? bare : null
+}
+
+/** The flag with its value: `--max-time 5`, `--max-time=5`, `-m5`. */
+function flagValue(words, names) {
+  for (const [index, word] of words.entries()) {
+    for (const name of names) {
+      if (word === name) return `${name} ${words[index + 1] ?? ""}`.trim()
+      if (word.startsWith(`${name}=`)) return word
+      if (name.length === 2 && word.startsWith(name) && word.length > 2 && /\d/.test(word[2])) return word
+    }
+  }
+  return null
+}
+
+/** The words a URL is invoked with: a `-c` script's own words when the URL sits inside one, otherwise the argv. */
+function wordsAround(process, url) {
+  const argv = process.argv
+  const at = argv.findIndex((word) => word.includes(url))
+  if (at < 0 || !process.shellWrapper) return argv
+  const script = toolOf(argv).index + 2
+  if (at !== script) return argv
+  // The URL is a word inside the script string: the words of its own segment.
+  for (const segment of shellSegments(argv[at])) {
+    if (segment.text.includes(url)) return shellWords(segment.text)
+  }
+  return shellWords(argv[at])
+}
+
+function caps(words, wholeArgv) {
+  const joined = wholeArgv.join(" ")
+  const timeout = flagValue(words, TIMEOUT_FLAGS)
+    || (wholeArgv.some((word) => basename(word) === "timeout") ? `timeout ${wholeArgv[wholeArgv.findIndex((word) => basename(word) === "timeout") + 1] ?? ""}`.trim() : null)
+  const head = joined.match(/(?:^|[\s|/])head\s+(?:-\S+\s+)*-c\s+(\S+)/)
+  const size = flagValue(words, SIZE_FLAGS) || (head ? `head -c ${head[1]}` : null)
+  return {
+    timeout: { observed: Boolean(timeout), via: timeout },
+    sizeCap: { observed: Boolean(size), via: size },
+  }
+}
+
+/** Shell and Python comments blanked, quotes respected, line count kept. */
+function blankHashComments(text) {
+  return text.split("\n").map((line) => {
+    let quote = null
+    for (let i = 0; i < line.length; i += 1) {
+      const ch = line[i]
+      if (quote) {
+        if (ch === "\\") i += 1
+        else if (ch === quote) quote = null
+      } else if (ch === '"' || ch === "'") quote = ch
+      else if (ch === "#" && (i === 0 || /\s/.test(line[i - 1]))) return line.slice(0, i)
+    }
+    return line
+  }).join("\n")
+}
+
+function callOnLine(line, text) {
+  if (/(?<![\w.])fetch\s*\(/.test(line)) return "fetch"
+  if (/XMLHttpRequest/.test(line) || /XMLHttpRequest/.test(text)) return "XMLHttpRequest"
+  const call = line.match(/([A-Za-z_][\w.]*)\s*\([^()]*https?:\/\//)
+  return call ? call[1] : null
+}
+
+/**
+ * @param {{ path: string, kind: string, text: string }} file
+ * @param {Array} processes the process rows of the same file
+ * @returns {{ hosts: Array, notResolvable: Array }}
+ */
+export function extractHosts(file, processes = []) {
+  const text = file.kind === "qml" || file.kind === "js" ? blankComments(file.text) : file.kind === "shell" || file.kind === "python" ? blankHashComments(file.text) : file.text
+  const hosts = []
+  const notResolvable = []
+  for (const match of text.matchAll(URL)) {
+    const url = match[0]
+    const line = lineOf(text, match.index)
+    const scheme = url.startsWith("https") ? "https" : "http"
+    const host = hostOf(url)
+    if (!host) {
+      notResolvable.push({ file: file.path, line, kind: "host", text: url })
+      continue
+    }
+    const process = processes.find((row) => row.file === file.path && Array.isArray(row.argv) && row.argv.some((word) => word.includes(url)) && (row.declaredIn !== "shell" || row.line === line))
+    let tool = null
+    let words = []
+    let argv = []
+    if (process) {
+      argv = process.argv
+      words = wordsAround(process, url)
+      tool = toolOf(words).tool
+      if (tool) tool = basename(tool)
+    } else {
+      const lineText = text.split("\n")[line - 1] || ""
+      tool = callOnLine(lineText, text)
+    }
+    const { timeout, sizeCap } = caps(words, argv)
+    hosts.push({
+      host,
+      scheme,
+      file: file.path,
+      line,
+      tool,
+      timeout,
+      sizeCap,
+      flags: words.filter((word) => word.startsWith("-") && word !== "-"),
+      privateAddress: privateAddress(host),
+    })
+  }
+  return { hosts, notResolvable }
+}

package/tools/inspect/inspect.mjs

@@ -0,0 +1,170 @@
+// `omakit inspect <plugin-dir>`: what a plugin tree does, as observations.
+// Resolve the subject the way `submit` does, read its installable tree at
+// the commit, run the four extractors over every file inspect reads, run
+// the marketplace's own baseline through `verify` for the capabilities,
+// and build the document of docs/INSPECT.md. No verdict, and the one score
+// a position among listed plugins, never a grade: every
+// row is a fact the text shows, labelled observed, and the document ends
+// with what the method cannot see.
+//
+// Runs nothing from the tree, resolves no host, opens no socket, writes
+// nothing into the tree. The one thing it fetches is what `verify` fetches,
+// which under the local transport is nothing.
+
+import { join } from "node:path"
+import { resolveSubject, SubjectError } from "../subject/resolve.mjs"
+import { marketplaceBaselineSection } from "../marketplace/verify.mjs"
+import { consequence } from "../marketplace/preflight.mjs"
+import { requirePin } from "../marketplace/pin.mjs"
+import { omakitCacheDir } from "../marketplace/paths.mjs"
+import { walkSubject } from "./walk.mjs"
+import { extractProcesses } from "./processes.mjs"
+import { extractHosts } from "./hosts.mjs"
+import { extractWrites } from "./writes.mjs"
+import { extractTimers } from "./timers.mjs"
+import { extractFunctions } from "./functions.mjs"
+import { evaluatePatterns, overSize, PATTERNS, rankOf, SIZE, sizeScore } from "./patterns.mjs"
+
+export const METHOD = "static extraction, regular expressions over qml and shell; observed, not executed"
+
+/**
+ * What regular expressions over QML and shell cannot see, printed at the
+ * end of every report and carried verbatim in the document. A tree that
+ * shows none of the facts is "observed nothing of this kind", never clean,
+ * because of this list.
+ */
+export const NOT_VISIBLE = Object.freeze([
+  "commands built at run time",
+  "hosts and paths from variables, properties, config or the environment",
+  "scripts a command calls that inspect does not follow",
+  "components loaded from outside the tree",
+  "encoded or obfuscated content, and what a sh -c or eval string runs",
+])
+
+/** The failure codes that mean the target could not be read at all: exit 2, the contract's second status. */
+export const NOT_READABLE = Object.freeze(["subject-not-found", "not-a-git-repository", "commit-not-found", "nothing-to-inspect", "usage"])
+
+export class InspectError extends Error {
+  constructor(code, message, remedy = null) {
+    super(message)
+    this.name = "InspectError"
+    this.code = code
+    this.remedy = remedy
+  }
+}
+
+/**
+ * @param {{ repoRoot: string, target: string, offline?: boolean, allowDirty?: boolean, omakitVersion: string, onPhase?: (text: string) => void, cacheRoot?: string }} options
+ * @returns {Promise<object>} the document of docs/INSPECT.md
+ */
+export async function inspectPlugin({ repoRoot, target, offline = false, allowDirty = false, omakitVersion, onPhase = () => {}, cacheRoot = omakitCacheDir() }) {
+  let subject
+  try {
+    subject = resolveSubject(target, { cacheRoot, allowDirty })
+  } catch (error) {
+    if (error instanceof SubjectError) throw new InspectError(error.code, error.message)
+    throw error
+  }
+  onPhase("reading the installable tree")
+  const tree = walkSubject(subject)
+  const dir = subject.subdir ? join(subject.dir, subject.subdir) : subject.dir
+  if (!tree.manifest && !tree.manifestError) {
+    throw new InspectError("nothing-to-inspect", `no manifest.json at the root of ${dir}, so there is no plugin to inspect`, "Pass the directory that holds the plugin's manifest.json.")
+  }
+  const pluginId = typeof tree.manifest?.id === "string" ? tree.manifest.id.trim() : null
+
+  onPhase("reading processes, hosts, writes and timers")
+  const processes = []
+  const hosts = []
+  const writes = []
+  const timers = []
+  const functions = []
+  const notResolvable = []
+  for (const file of tree.files) {
+    // Each function carries its rank among the listed ones (M12).
+    functions.push(...extractFunctions(file).map((entry) => ({ ...entry, percentile: rankOf(entry) })))
+    const rows = extractProcesses(file)
+    processes.push(...rows)
+    for (const row of rows) {
+      if (row.argvForm === "computed") notResolvable.push({ file: row.file, line: row.line, kind: "command", text: `command: ${row.commandText}` })
+    }
+    const found = extractHosts(file, rows)
+    hosts.push(...found.hosts)
+    notResolvable.push(...found.notResolvable)
+    writes.push(...extractWrites(file, { pluginId }))
+    const ticking = extractTimers(file)
+    timers.push(...ticking.timers)
+    notResolvable.push(...ticking.notResolvable)
+  }
+
+  let marketplaceBaseline
+  let blockingRules = []
+  if (offline) {
+    marketplaceBaseline = { skipped: true, reason: "--offline" }
+  } else {
+    onPhase("running the official security baseline over a local snapshot")
+    // The baseline sees the plugin's tree and nothing around it: for a plugin
+    // kept below the root of a larger repository, the local transport serves
+    // that directory's tree as the whole tree. Without this the baseline
+    // scanned the repository root and reported the root's evidence as the
+    // plugin's, which is the 0.1 Passport's first failure.
+    marketplaceBaseline = await marketplaceBaselineSection({ repoRoot, subject, subdir: subject.subdir })
+    if (marketplaceBaseline.invoked && marketplaceBaseline.official && !marketplaceBaseline.official.error) {
+      blockingRules = (await consequence(requirePin(repoRoot).dir, marketplaceBaseline.official)).selectivelyBlockingRules
+    }
+  }
+
+  const facts = { processes, hosts, writes, timers, notResolvable, files: tree.files, readme: tree.readme, baseline: marketplaceBaseline, blockingRules }
+  const { patterns, lookedFor } = evaluatePatterns(facts)
+
+  return {
+    omakit: omakitVersion,
+    command: "inspect",
+    method: METHOD,
+    subject: {
+      dir,
+      commit: subject.commit,
+      repository: { url: subject.repository.url },
+      mode: subject.mode,
+      pluginId,
+      filesRead: tree.filesRead,
+    },
+    observed: { processes, hosts, writes, timers, functions },
+    // Size: the functions over the M12 thresholds, longest first. A count of
+    // lines, branches and nesting over the text, compared with what 90 of
+    // 100 functions in listed trees stay under; never a judgement.
+    size: {
+      measurement: SIZE.measurement,
+      sample: { trees: SIZE.trees, functions: SIZE.functions },
+      thresholds: { lines: SIZE.lines, branches: SIZE.branches, depth: SIZE.depth },
+      // 10 minus the mean rank of this tree's functions among the listed
+      // ones: where the tree sits, never whether it is good; null with no
+      // function to rank.
+      score: sizeScore(functions),
+      over: overSize(functions),
+    },
+    // The headline split: a shell script contributes one site per command
+    // segment, so a tree with a few scripts carries hundreds of process
+    // sites beside a handful of QML Process blocks, and the two are said
+    // apart wherever the count is printed.
+    counts: {
+      processes: {
+        total: processes.length,
+        qml: processes.filter((row) => row.declaredIn === "qml").length,
+        shell: processes.filter((row) => row.declaredIn === "shell").length,
+      },
+      hosts: hosts.length,
+      writes: writes.length,
+      timers: timers.length,
+      functions: functions.length,
+      notResolvable: notResolvable.length,
+    },
+    notResolvable,
+    patterns,
+    lookedFor,
+    notVisible: [...NOT_VISIBLE],
+    marketplaceBaseline,
+  }
+}
+
+export { PATTERNS }