forgedock 1.2.0 → 1.3.0

package/packages/protocol/src/cli.js

@@ -42,10 +42,10 @@
 
 import { readFileSync, readdirSync } from 'node:fs';
 import { resolve, extname } from 'node:path';
-import { createHash } from 'node:crypto';
 import { parse } from './parse.js';
 import { validate } from './validate.js';
 import { emit } from './emit.js';
+import { encodeCard, decodeCardInlineValue } from './card.js';
 
 // ---------------------------------------------------------------------------
 // Argv routing
@@ -114,16 +114,15 @@ function cmdEmit(argv) {
     // Payload: canonical JSON (sorted keys, single line, UTF-8).
     // The Base64url alphabet [A-Za-z0-9_-] cannot contain '>', '<', or '!' —
     // so -->, --!>, and <!-- are structurally unrepresentable by construction.
+    // Codec lives in ./card.js (shared with validate.js and the test suite).
     const payload = Object.keys(fields).length > 0 ? { ...fields } : {};
     if (type) payload.type = type.toUpperCase();
-    const canonical = canonicalJson(payload);
-    const b64url = toBase64url(canonical);
-    const sha8 = createHash('sha256').update(canonical, 'utf8').digest('hex').slice(0, 8);
+    const { inlineValue } = encodeCard(payload);
     // The colon-separator form "<!-- FORGE:CARD: v1 sha:X b64:Y -->" matches
     // OPENING_TAG_RE so parse() can extract the inline value "v1 sha:X b64:Y".
     // This keeps the canonical FORGE inline-value syntax (§3.4) while adding
     // the sha8 integrity prefix and Base64url payload from the design decision.
-    process.stdout.write(`<!-- FORGE:CARD: v1 sha:${sha8} b64:${b64url} -->\n`);
+    process.stdout.write(`<!-- FORGE:CARD: ${inlineValue} -->\n`);
     return;
   }
 
@@ -351,75 +350,3 @@ function die(msg) {
   process.stderr.write(`forge-protocol-cli: ${msg}\n`);
   process.exit(1);
 }
-
-/**
- * Produce canonical JSON: sorted keys, single line, UTF-8.
- * This is the payload that gets Base64url-encoded in the CARD format.
- */
-function canonicalJson(obj) {
-  return JSON.stringify(sortKeysDeep(obj));
-}
-
-function sortKeysDeep(val) {
-  if (Array.isArray(val)) return val.map(sortKeysDeep);
-  if (val !== null && typeof val === 'object') {
-    return Object.fromEntries(
-      Object.keys(val)
-        .sort()
-        .map(k => [k, sortKeysDeep(val[k])]),
-    );
-  }
-  return val;
-}
-
-/**
- * Encode a UTF-8 string as Base64url (URL-safe alphabet, no padding).
- * The Base64url alphabet [A-Za-z0-9_-] cannot contain '>', '<', or '!' —
- * so -->, --!>, and <!-- are structurally unrepresentable by construction
- * (design decision 2026-07-08: encoding beats escaping).
- *
- * @param {string} str — UTF-8 string to encode
- * @returns {string} Base64url-encoded string without padding
- */
-function toBase64url(str) {
-  return Buffer.from(str, 'utf8')
-    .toString('base64')
-    .replace(/\+/g, '-')
-    .replace(/\//g, '_')
-    .replace(/=/g, '');
-}
-
-/**
- * Decode a CARD inline value of the form "v1 sha:<sha8hex> b64:<base64url>"
- * and return the parsed JSON object, or null if the format is unrecognized.
- * Verifies the sha8 integrity prefix; returns null on corruption or tampering.
- *
- * @param {string} inlineValue — the parsed inlineValue from a CARD annotation
- * @returns {object|null}
- */
-function decodeCardInlineValue(inlineValue) {
-  const m = inlineValue.match(/^v1\s+sha:([0-9a-f]{8})\s+b64:([A-Za-z0-9_-]+)$/);
-  if (!m) return null;
-  const [, sha8, b64url] = m;
-
-  // Restore standard Base64 padding and swap the URL-safe alphabet back.
-  const b64std = b64url.replace(/-/g, '+').replace(/_/g, '/');
-  const padded = b64std + '='.repeat((4 - (b64std.length % 4)) % 4);
-
-  let canonical;
-  try {
-    canonical = Buffer.from(padded, 'base64').toString('utf8');
-  } catch {
-    return null;
-  }
-
-  // Integrity check: first 8 hex chars of SHA-256 must match the prefix.
-  const expectedSha8 = createHash('sha256').update(canonical, 'utf8').digest('hex').slice(0, 8);
-  if (expectedSha8 !== sha8) return null;
-
-  try {
-    return JSON.parse(canonical);
-  } catch {
-    return null;
-  }
-}

package/packages/protocol/src/emit.js

@@ -56,15 +56,26 @@ function sanitizeFieldValue(rawValue) {
       `emit(): field value resolves to a bare FORGE tag/sentinel line, which is not permitted: ${JSON.stringify(rawValue)}`,
     );
   }
+  // Escape "&" FIRST, before either delimiter-escape pass below. Without this,
+  // a value that already contains literal entity-like text (e.g. "<!--&gt;"
+  // or "&lt;!--") collides with the encoded form of an unrelated real
+  // delimiter — sanitizeFieldValue() would not be injective, and the
+  // collision is unrecoverable by unescapeFieldValue() on decode
+  // (forge#2137). Escaping "&" first guarantees every "&lt;!--"/"--&gt;"/
+  // "--!&gt;" sequence in the output was produced by *this* pass, never by
+  // pre-existing entity text (which is now doubly-escaped, e.g.
+  // "&" -> "&amp;", left untouched by the delimiter passes that only match
+  // literal "<!--"/"--"+">" character sequences).
+  const withAmpEscaped = folded.replace(/&/g, '&amp;');
   // Escape the HTML comment opener ("<!--") so a value cannot start a new
   // unterminated HTML comment in GitHub's renderer, which would visually swallow
   // subsequent lines — the same rendering-leak the closer escaping below prevents
   // from the opposite direction. (forge#1638)
-  const withOpenerEscaped = folded.replace(/<!--/g, '&lt;!--');
+  const withOpenerEscaped = withAmpEscaped.replace(/<!--/g, '&lt;!--');
   // Escape both HTML comment-close forms ("-->" and "--!>") — only replacing the
   // literal character sequence (not just documenting the risk) prevents the
-  // enclosing comment from terminating early. (forge#1594; noted but never fixed
-  // for the FORGE:STATE codec in bin/engine/state.mjs — see issue investigation)
+  // enclosing comment from terminating early. (forge#1594; the FORGE:STATE codec
+  // in bin/engine/state.mjs now mirrors this full three-form scheme — forge#2119)
   return withOpenerEscaped.replace(/--(!)?>/g, (_, bang) => `--${bang || ''}&gt;`);
 }
 

package/packages/protocol/src/index.js

@@ -1,7 +1,7 @@
 /**
  * @forgedock/protocol — FORGE Annotation Protocol v1.0 reference implementation.
  *
- * Provides parse, validate, and emit for all 13 reserved annotation types
+ * Provides parse, validate, and emit for all reserved annotation types
  * defined in the FORGE Annotation Protocol specification (CC-BY-4.0).
  *
  * @example
@@ -29,6 +29,7 @@ export { parse } from './parse.js';
 export { validate } from './validate.js';
 export { emit, emitPartial, isKnownType } from './emit.js';
 export { RESERVED_TYPES, RESERVED_TYPE_NAMES, Category, SentinelState } from './types.js';
+export { canonicalJson, toBase64url, encodeCard, decodeCardInlineValue } from './card.js';
 
 /**
  * @typedef {Object} ParsedAnnotation

package/packages/protocol/src/parse.js

@@ -101,17 +101,25 @@ function escapeRegExp(str) {
 /**
  * Reverse the HTML entity escapes applied by emit()'s sanitizeFieldValue().
  *
- * emit() escapes three HTML comment delimiter sequences to protect GitHub's
- * renderer from comment injection:
+ * emit() escapes "&" first, then three HTML comment delimiter sequences, to
+ * protect GitHub's renderer from comment injection AND to keep the encoding
+ * injective (reversible for every input, not just delimiter-free ones):
+ *   0. `&`     → `&`      (forge#2137 — must run first on encode)
  *   1. `<!--`  → `&lt;!--`   (forge#1638)
  *   2. `--!>`  → `--!&gt;`   (forge#1594)
  *   3. `-->`   → `--&gt;`    (forge#1594)
  *
- * parse() must apply the inverse so that round-tripping through emit → parse
- * is lossless. Unescape order: `--!&gt;` before `--&gt;` (most-specific first)
- * to avoid the `!` being left dangling by an overly-greedy `--&gt;` pass.
- * The `&lt;!--` unescape is independent and can run in any relative order.
- * (forge#1662)
+ * parse() must apply the exact inverse, in reverse order, so that
+ * round-tripping through emit → parse is lossless. Unescape order:
+ * `--!&gt;` before `--&gt;` (most-specific first) to avoid the `!` being left
+ * dangling by an overly-greedy `--&gt;` pass; `&lt;!--` can run in any order
+ * relative to those two. The `&amp;` → `&` unescape MUST run LAST — doing it
+ * first (or interleaved) would corrupt the `&` that is part of a freshly
+ * restored `<!--`/`-->`/`--!>` sequence. Running "&" first on encode and last
+ * on decode is what makes the scheme injective: any pre-existing entity-like
+ * text in the original value (e.g. `&lt;!--` or `<!--&gt;`) is doubly-escaped
+ * at encode time and can never collide with the single-escaped form produced
+ * by a real delimiter (forge#2137; forge#1662).
  *
  * @param {string} raw
  * @returns {string}
@@ -120,7 +128,8 @@ function unescapeFieldValue(raw) {
   return raw
     .replace(/--!&gt;/g, '--!>')
     .replace(/--&gt;/g, '-->')
-    .replace(/&lt;!--/g, '<!--');
+    .replace(/&lt;!--/g, '<!--')
+    .replace(/&amp;/g, '&');
 }
 
 /**

package/packages/protocol/src/types.js

@@ -18,7 +18,7 @@ export const SentinelState = {
 };
 
 /**
- * All 16 reserved annotation types from spec §4.
+ * All reserved annotation types from spec §4.
  * Spec ref: §4.1 lifecycle, §4.2 cross-artifact, §4.3 control/error markers.
  * Added CLAIM and CLAIM_RELEASED in forge#1736 — claims board for claim-level parallelism.
  * Added AUTOPILOT_CYCLE in forge#1753 — durable cycle state for /autopilot.

package/packages/protocol/src/validate.js

@@ -8,12 +8,14 @@
  *   - Required field presence (§4.1 per-type field tables)
  *   - Required field enum values (Verdict, Confidence, Severity, Task Type)
  *   - Inline-value non-empty (§3.4)
+ *   - CARD inline-value decodability + sha8 integrity (§4.2 — see card.js)
  *   - Partial sentinel marking (§3.3) — generates a warning, not an error
  *
  * @license MIT
  */
 
 import { RESERVED_TYPES, SentinelState } from './types.js';
+import { decodeCardInlineValue } from './card.js';
 
 /**
  * Validate a parsed annotation against the spec.
@@ -46,6 +48,15 @@ export function validate(annotation) {
   if (typeDef.inlineValue) {
     if (inlineValue === null || inlineValue === '') {
       errors.push(`Inline-value annotation "${type}" must carry a non-empty value (§3.4)`);
+    } else if (type === 'CARD') {
+      // CARD-specific (§4.2): the inline value must decode via the Base64url
+      // codec and pass its sha8 integrity check. A malformed encoding or a
+      // tampered/corrupted payload must not be treated as a valid annotation.
+      if (decodeCardInlineValue(inlineValue) === null) {
+        errors.push(
+          `Inline-value "CARD" annotation failed to decode or failed its sha8 integrity check (§4.2)`,
+        );
+      }
     }
     return { valid: errors.length === 0, errors, warnings };
   }

package/scripts/check-command-side-effects.sh

@@ -0,0 +1,261 @@
+#!/usr/bin/env bash
+# SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+# SPDX-License-Identifier: AGPL-3.0-or-later
+#
+# check-command-side-effects.sh — Scan commands/**/*.md for two classes of
+#                                  spec-as-code side-effect defects.
+#
+# Command specs are spec-as-code: they drive autonomous side effects (auto-merge,
+# gh gist create, git push, label mutations, issue creation). This script catches
+# two defect classes before they reach staging:
+#
+#   Class A — Unconditionally prohibited patterns (full-corpus, always blocked):
+#     `gh gist create --public` or `gh gist edit --public` anywhere in a code block.
+#     Gists created by the pipeline MUST be secret — --public exposes private repo
+#     titles, root causes, and file paths to the world. (Ref: forge#1587)
+#
+#   Class B — Side-effect verbs in sections with no DRY_RUN/governor guard (diff-aware):
+#     When added lines introduce a side-effect verb (gh issue create, gh pr merge,
+#     git push, gh issue edit|comment, --auto-merge, --add/remove-label) inside a
+#     code block in a section that has no guard expression (DRY_RUN, GOVERNOR,
+#     --dry-run) anywhere in that section's code blocks, the change is flagged.
+#     Operates on the diff (GITHUB_BASE_SHA or HEAD^) to avoid flagging legacy corpus.
+#     (Ref: forge#1609 — signal-planner.md DRY_RUN guard placed after the create it guards)
+#
+# Allowlist: add <!-- allowlist:check-command-side-effects --> on the same line as
+# the side-effect verb to suppress that specific hit.
+#
+# Usage:
+#   check-command-side-effects.sh [--full] [<commands_dir>]
+#     --full         Scan entire corpus (not just diff) for Class A violations only.
+#                    Class B is diff-aware by design — --full does not change Class B.
+#     commands_dir   Path to the commands/ directory (default: ./commands)
+#
+# Environment:
+#   GITHUB_BASE_SHA   When set, used as the base commit for Class B diff mode.
+#
+# Exit codes:
+#   0  no violations found
+#   1  one or more violations found (listed to stderr)
+#   2  usage / dependency error
+#
+# <!-- Added: forge#1609 -->
+
+set -euo pipefail
+
+# ---------------------------------------------------------------------------
+# Argument parsing
+# ---------------------------------------------------------------------------
+
+FULL_MODE=0
+COMMANDS_DIR="./commands"
+
+for arg in "$@"; do
+  case "$arg" in
+    -h|--help)
+      sed -n '2,/^set -/p' "$0" | grep '^#' | sed 's/^# *//'
+      exit 0
+      ;;
+    --full) FULL_MODE=1 ;;
+    -*) echo "ERROR: unknown option: $arg" >&2; exit 2 ;;
+    *) COMMANDS_DIR="$arg" ;;
+  esac
+done
+
+if [ ! -d "$COMMANDS_DIR" ]; then
+  echo "ERROR: commands directory not found: $COMMANDS_DIR" >&2
+  echo "Usage: $0 [--full] [<commands_dir>]" >&2
+  exit 2
+fi
+
+# ---------------------------------------------------------------------------
+# Constants
+# ---------------------------------------------------------------------------
+
+# Code fence marker — stored in variable to avoid backtick shell-expansion bugs.
+# Do NOT inline as grep -E '^[[:space:]]*\`\`\`' — escaped backticks in grep
+# patterns undergo shell expansion, turning the pattern into '^[[:space:]]*'
+# which matches every line. (forge#1609)
+FENCE='```'
+
+# Guard expressions: any occurrence in a section's code blocks marks it as guarded
+GUARD_PATTERN='DRY_RUN|GOVERNOR|--dry-run|DRY_RUN_MODE|DryRun'
+
+# Class B side-effect verbs (in code blocks)
+SIDE_EFFECT_PATTERN='gh[[:space:]]+(issue[[:space:]]+(create|edit|comment)|pr[[:space:]]+(merge|create|edit)|gist[[:space:]]+(create|edit))|git[[:space:]]+push|--auto-merge|--add-label|--remove-label'
+
+ALLOWLIST_TOKEN='allowlist:check-command-side-effects'
+
+VIOLATIONS=0
+
+# ---------------------------------------------------------------------------
+# Class A: Scan for gh gist create/edit --public in code blocks (full corpus)
+# This runs regardless of --full or diff mode.
+# ---------------------------------------------------------------------------
+
+class_a_scan() {
+  local file="$1"
+  local IN_CB=0
+  local HAS_GIST=0
+  local BLOCK_LINES=""
+  local BLOCK_START=0
+  local LINENO=0
+  local line PUBLIC_LINENO ACTUAL_LINE
+
+  while IFS= read -r line; do
+    LINENO=$((LINENO + 1))
+
+    if echo "$line" | grep -qF "$FENCE"; then
+      if [ "$IN_CB" -eq 0 ]; then
+        IN_CB=1
+        HAS_GIST=0
+        BLOCK_LINES=""
+        BLOCK_START=$LINENO
+      else
+        # End of code block — check if it had both gh gist and --public
+        if [ "$HAS_GIST" -eq 1 ] && echo "$BLOCK_LINES" | grep -qE '^[[:space:]]*--public([[:space:]]|$)'; then
+          # Find the line number of --public within the block
+          PUBLIC_LINENO=$(echo "$BLOCK_LINES" | grep -n '^[[:space:]]*--public' | head -1 | cut -d: -f1)
+          ACTUAL_LINE=$((BLOCK_START + PUBLIC_LINENO))
+          if ! echo "$BLOCK_LINES" | grep -qF "$ALLOWLIST_TOKEN"; then
+            echo "HIGH | $file | line ~$ACTUAL_LINE | Class A: 'gh gist create/edit --public' in code block — gists MUST be secret (omit --public); --public exposes private repo data. (forge#1587)" >&2
+            VIOLATIONS=$((VIOLATIONS + 1))
+          fi
+        fi
+        IN_CB=0
+        HAS_GIST=0
+        BLOCK_LINES=""
+      fi
+      continue
+    fi
+
+    if [ "$IN_CB" -eq 1 ]; then
+      BLOCK_LINES="${BLOCK_LINES}${line}
+"
+      # Check if this line has gh gist create/edit (even with line continuation \)
+      if echo "$line" | grep -qE 'gh[[:space:]]+gist[[:space:]]+(create|edit)'; then
+        HAS_GIST=1
+      fi
+    fi
+  done < "$file"
+}
+
+while IFS= read -r file; do
+  [ -f "$file" ] || continue
+  class_a_scan "$file"
+done < <(find "$COMMANDS_DIR" -name '*.md' | sort)
+
+# ---------------------------------------------------------------------------
+# Class B: Diff-aware scan for unguarded side-effect verbs in added lines
+# ---------------------------------------------------------------------------
+
+# Determine base SHA for diff
+BASE_SHA=""
+if [ -n "${GITHUB_BASE_SHA:-}" ]; then
+  BASE_SHA="$GITHUB_BASE_SHA"
+elif [ -n "${GITHUB_EVENT_PULL_REQUEST_BASE_SHA:-}" ]; then
+  BASE_SHA="$GITHUB_EVENT_PULL_REQUEST_BASE_SHA"
+else
+  BASE_SHA="$(git rev-parse HEAD^ 2>/dev/null || echo '')"
+fi
+
+if [ -z "$BASE_SHA" ]; then
+  echo "INFO: No git base SHA available — Class B (diff-aware) check skipped" >&2
+else
+  # Get list of changed command spec files
+  CHANGED_SPECS=$(git diff --name-only "$BASE_SHA"...HEAD -- "${COMMANDS_DIR}" 2>/dev/null \
+    | grep -E '\.md$' | grep -v '^$' || true)
+
+  if [ -z "$CHANGED_SPECS" ]; then
+    echo "OK (Class B): No commands/*.md files changed — diff check skipped"
+  else
+    echo "Class B: Checking changed spec files for unguarded side-effect verbs:"
+    echo "$CHANGED_SPECS"
+    echo ""
+
+    while IFS= read -r file; do
+      [ -f "$file" ] || continue
+
+      # Get added lines for this file
+      ADDED_CONTENT=$(git diff "$BASE_SHA"...HEAD -- "$file" 2>/dev/null \
+        | grep '^+' | grep -v '^+++' | sed 's/^+//' || true)
+
+      # Skip if no added lines contain side-effect verbs
+      if ! echo "$ADDED_CONTENT" | grep -qE "$SIDE_EFFECT_PATTERN"; then
+        continue
+      fi
+
+      # Parse the full file to map sections → (has_guard, has_side_effect_in_added_lines)
+      IN_CB=0
+      SECTION="(top)"
+      SECTION_HAS_GUARD=0
+      SECTION_HAS_ADDED_SE=0
+      SECTION_SE_LINE=0
+      SECTION_SE_VERB=""
+      LINENO=0
+
+      flush_and_reset() {
+        local new_heading="$1"
+        if [ "$SECTION_HAS_ADDED_SE" -eq 1 ] && [ "$SECTION_HAS_GUARD" -eq 0 ]; then
+          echo "HIGH | $file | line $SECTION_SE_LINE | Class B: side-effect '$SECTION_SE_VERB' added to section '$SECTION' which has no DRY_RUN/governor guard — add guard before this line or wrap in DRY_RUN check" >&2
+          VIOLATIONS=$((VIOLATIONS + 1))
+        fi
+        SECTION="$new_heading"
+        SECTION_HAS_GUARD=0
+        SECTION_HAS_ADDED_SE=0
+        SECTION_SE_LINE=0
+        SECTION_SE_VERB=""
+        IN_CB=0
+      }
+
+      while IFS= read -r line; do
+        LINENO=$((LINENO + 1))
+
+        # Section heading reset
+        if echo "$line" | grep -qE '^#{1,4}[[:space:]]+'; then
+          heading=$(echo "$line" | sed 's/^#*[[:space:]]*//' | sed 's/[[:space:]]*$//')
+          flush_and_reset "$heading"
+          continue
+        fi
+
+        # Code block fence
+        if echo "$line" | grep -qF "$FENCE"; then
+          [ "$IN_CB" -eq 0 ] && IN_CB=1 || IN_CB=0
+          continue
+        fi
+
+        if [ "$IN_CB" -eq 1 ]; then
+          if echo "$line" | grep -qF "$ALLOWLIST_TOKEN"; then continue; fi
+
+          # Check for guard
+          echo "$line" | grep -qE "$GUARD_PATTERN" && SECTION_HAS_GUARD=1
+
+          # Check if this line is in the diff's added lines AND has a side-effect verb
+          if [ "$SECTION_HAS_ADDED_SE" -eq 0 ] && echo "$line" | grep -qE "$SIDE_EFFECT_PATTERN"; then
+            # Is this specific line in the added content?
+            if echo "$ADDED_CONTENT" | grep -qF "${line:0:80}" 2>/dev/null; then
+              SECTION_HAS_ADDED_SE=1
+              SECTION_SE_LINE=$LINENO
+              SECTION_SE_VERB=$(echo "$line" | grep -oE "$SIDE_EFFECT_PATTERN" | head -1 || echo "side-effect")
+            fi
+          fi
+        fi
+      done < "$file"
+
+      flush_and_reset "(end-of-file)"
+
+    done <<< "$CHANGED_SPECS"
+  fi
+fi
+
+# ---------------------------------------------------------------------------
+# Summary
+# ---------------------------------------------------------------------------
+
+if [ "$VIOLATIONS" -gt 0 ]; then
+  echo "check-command-side-effects: $VIOLATIONS violation(s) found. See stderr for details." >&2
+  exit 1
+fi
+
+echo "OK: No side-effect violations found"
+exit 0