@codyswann/lisa 2.145.2 → 2.146.0

package/plugins/lisa-wiki-cursor/scripts/ingest_slack_channel.py

@@ -22,10 +22,37 @@ from typing import Any
 
 
 TOKEN_PATTERNS = [
-    re.compile(r"xox[pbar]-[A-Za-z0-9-]+"),
-    re.compile(r"(?i)bearer\s+[A-Za-z0-9._~+/=-]{20,}"),
-    re.compile(r"AKIA[0-9A-Z]{16}"),
-    re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----.*?-----END [A-Z ]*PRIVATE KEY-----", re.S),
+    (re.compile(r"xox[pbar]-[A-Za-z0-9-]+"), "[REDACTED:OAUTH_TOKEN]"),
+    (
+        re.compile(r"(?i)bearer\s+[A-Za-z0-9._~+/=-]{20,}"),
+        "[REDACTED:OAUTH_TOKEN]",
+    ),
+    (re.compile(r"AKIA[0-9A-Z]{16}"), "[REDACTED:API_KEY]"),
+    (
+        re.compile(
+            r"-----BEGIN [A-Z ]*PRIVATE KEY-----.*?-----END [A-Z ]*PRIVATE KEY-----",
+            re.S,
+        ),
+        "[REDACTED:PRIVATE_KEY]",
+    ),
+    (
+        re.compile(r"\b(?!000|666|9\d\d)\d{3}-(?!00)\d{2}-(?!0000)\d{4}\b"),
+        "[REDACTED:SSN]",
+    ),
+    (
+        re.compile(
+            r"\b(?:password|passwd|pwd)\s*[:=]\s*(['\"]?)([^\s'\",;]{8,})\1",
+            re.I,
+        ),
+        "[REDACTED:PASSWORD]",
+    ),
+    (
+        re.compile(
+            r"\b(?:api[_-]?key|access[_-]?key|secret[_-]?key|client[_-]?secret)\s*[:=]\s*(['\"]?)([A-Za-z0-9._-]{20,})\1",
+            re.I,
+        ),
+        "[REDACTED:API_KEY]",
+    ),
 ]
 
 
@@ -49,8 +76,13 @@ def ts_from_input(value: str | None) -> str | None:
 
 def redact(text: str) -> str:
     out = text
-    for pattern in TOKEN_PATTERNS:
-        out = pattern.sub("[REDACTED]", out)
+    for pattern, replacement in TOKEN_PATTERNS:
+        out = pattern.sub(
+            lambda match: match.group(0).replace(match.group(2), replacement)
+            if len(match.groups()) >= 2
+            else replacement,
+            out,
+        )
     return out
 
 
@@ -289,7 +321,7 @@ def main() -> int:
             for reply in replies:
                 lines.extend(render_message(reply))
 
-    source_path.write_text("\n".join(lines), encoding="utf-8")
+    source_path.write_text(redact("\n".join(lines)), encoding="utf-8")
 
     latest_ts = previous_state.get("latest_message_ts")
     if messages:

package/plugins/src/wiki/scripts/_wiki-lib.mjs

@@ -5,6 +5,7 @@
  */
 import fs from "node:fs";
 import path from "node:path";
+import { sanitizeWikiSourceText } from "./wiki-safety.mjs";
 
 /** Read and parse a JSON file, or return undefined if missing/invalid. */
 export function readJsonSafe(file) {
@@ -59,6 +60,24 @@ export function walkFiles(dir, { ext } = {}) {
   return out.sort();
 }
 
+/**
+ * Sanitize a wiki source note immediately before it is persisted.
+ *
+ * Connectors should keep fetched raw material in memory or temporary locations
+ * outside the repo, render the source note, then call this helper at the final
+ * `wiki/sources/**` write boundary. The return value contains safe finding
+ * metadata only; callers can include it in handoff metadata when useful.
+ */
+export function writeSanitizedSourceNote(file, rawText, sourceMetadata = {}) {
+  const result = sanitizeWikiSourceText(rawText, {
+    ...sourceMetadata,
+    path: sourceMetadata.path ?? file,
+  });
+  fs.mkdirSync(path.dirname(file), { recursive: true });
+  fs.writeFileSync(file, result.text);
+  return result;
+}
+
 /**
  * Minimal frontmatter detector. Returns whether a leading `--- ... ---` block
  * exists and the top-level keys it declares (enough to check required fields;

package/plugins/src/wiki/scripts/ingest-git.mjs

@@ -15,7 +15,7 @@
 import fs from "node:fs";
 import path from "node:path";
 import { execFileSync } from "node:child_process";
-import { readJsonSafe, SECRET_PATTERNS } from "./_wiki-lib.mjs";
+import { readJsonSafe, writeSanitizedSourceNote } from "./_wiki-lib.mjs";
 
 const argv = process.argv.slice(2);
 const opt = (n, d) => {
@@ -53,12 +53,6 @@ const commitExists = c => {
     return false;
   }
 };
-const redact = t =>
-  SECRET_PATTERNS.reduce(
-    (acc, { re }) => acc.replace(new RegExp(re, "g"), "[REDACTED]"),
-    t
-  );
-
 if (
   !fs.existsSync(path.join(repo, ".git")) &&
   !tryGit(["rev-parse", "--is-inside-work-tree"])
@@ -165,8 +159,11 @@ ${
 }
 `;
 
-fs.mkdirSync(sourceDir, { recursive: true });
-fs.writeFileSync(notePath, redact(note));
+const safety = writeSanitizedSourceNote(notePath, note, {
+  sourceId: path.relative(process.cwd(), notePath),
+  sourceSystem: "git",
+  project: slug,
+});
 
 const meta = {
   connector: "git",
@@ -174,6 +171,10 @@ const meta = {
   ranAt: new Date().toISOString(),
   proposedCursor: { lastCommit: head, lastPr },
   sourceNotes: [path.relative(process.cwd(), notePath)],
+  safety: {
+    reviewRequired: safety.reviewRequired,
+    findings: safety.findings,
+  },
 };
 if (emitMeta) {
   fs.mkdirSync(path.dirname(emitMeta), { recursive: true });

package/plugins/src/wiki/scripts/ingest-memory.mjs

@@ -15,7 +15,11 @@
 import fs from "node:fs";
 import path from "node:path";
 import os from "node:os";
-import { loadConfig, walkFiles, SECRET_PATTERNS } from "./_wiki-lib.mjs";
+import {
+  loadConfig,
+  walkFiles,
+  writeSanitizedSourceNote,
+} from "./_wiki-lib.mjs";
 
 const argv = process.argv.slice(2);
 const opt = (n, d) => {
@@ -79,15 +83,10 @@ if (!(under(claudeMem) || under(projectCodexMem) || allowedRoots.some(under))) {
   );
 }
 
-const redact = t =>
-  SECRET_PATTERNS.reduce(
-    (acc, { re }) => acc.replace(new RegExp(re, "g"), "[REDACTED]"),
-    t
-  );
 const mdFiles = walkFiles(resolvedMem, { ext: ".md" });
 const date = new Date().toISOString().slice(0, 10);
 const entries = mdFiles.map(f => {
-  const body = redact(fs.readFileSync(f, "utf8")).trim();
+  const body = fs.readFileSync(f, "utf8").trim();
   return `### ${path.basename(f)}\n\n${body}`;
 });
 
@@ -110,8 +109,10 @@ sensitivity: internal
 ${entries.join("\n\n") || "_(no memory files)_"}
 `;
 
-fs.mkdirSync(sourceDir, { recursive: true });
-fs.writeFileSync(notePath, note);
+const safety = writeSanitizedSourceNote(notePath, note, {
+  sourceId: path.relative(process.cwd(), notePath),
+  sourceSystem: "memory",
+});
 
 const meta = {
   connector: "memory",
@@ -119,6 +120,10 @@ const meta = {
   ranAt: new Date().toISOString(),
   proposedCursor: { files: mdFiles.length, lastIngest: date },
   sourceNotes: [path.relative(process.cwd(), notePath)],
+  safety: {
+    reviewRequired: safety.reviewRequired,
+    findings: safety.findings,
+  },
 };
 if (emitMeta) {
   fs.mkdirSync(path.dirname(emitMeta), { recursive: true });

package/plugins/src/wiki/scripts/ingest-roles.mjs

@@ -10,7 +10,11 @@
  */
 import fs from "node:fs";
 import path from "node:path";
-import { loadConfig, walkFiles } from "./_wiki-lib.mjs";
+import {
+  loadConfig,
+  walkFiles,
+  writeSanitizedSourceNote,
+} from "./_wiki-lib.mjs";
 
 const argv = process.argv.slice(2);
 const opt = (n, d) => {
@@ -61,8 +65,10 @@ ${rosterRows}
 ${staffPages.length ? staffPages.map(p => `- \`${path.relative(wikiRoot, p)}\``).join("\n") : "_(none)_"}
 `;
 
-fs.mkdirSync(sourceDir, { recursive: true });
-fs.writeFileSync(notePath, note);
+const safety = writeSanitizedSourceNote(notePath, note, {
+  sourceId: path.relative(process.cwd(), notePath),
+  sourceSystem: "roles",
+});
 
 const meta = {
   connector: "roles",
@@ -74,6 +80,10 @@ const meta = {
     lastIngest: date,
   },
   sourceNotes: [path.relative(process.cwd(), notePath)],
+  safety: {
+    reviewRequired: safety.reviewRequired,
+    findings: safety.findings,
+  },
 };
 if (emitMeta) {
   fs.mkdirSync(path.dirname(emitMeta), { recursive: true });

package/plugins/src/wiki/scripts/ingest_slack_channel.py

@@ -22,10 +22,37 @@ from typing import Any
 
 
 TOKEN_PATTERNS = [
-    re.compile(r"xox[pbar]-[A-Za-z0-9-]+"),
-    re.compile(r"(?i)bearer\s+[A-Za-z0-9._~+/=-]{20,}"),
-    re.compile(r"AKIA[0-9A-Z]{16}"),
-    re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----.*?-----END [A-Z ]*PRIVATE KEY-----", re.S),
+    (re.compile(r"xox[pbar]-[A-Za-z0-9-]+"), "[REDACTED:OAUTH_TOKEN]"),
+    (
+        re.compile(r"(?i)bearer\s+[A-Za-z0-9._~+/=-]{20,}"),
+        "[REDACTED:OAUTH_TOKEN]",
+    ),
+    (re.compile(r"AKIA[0-9A-Z]{16}"), "[REDACTED:API_KEY]"),
+    (
+        re.compile(
+            r"-----BEGIN [A-Z ]*PRIVATE KEY-----.*?-----END [A-Z ]*PRIVATE KEY-----",
+            re.S,
+        ),
+        "[REDACTED:PRIVATE_KEY]",
+    ),
+    (
+        re.compile(r"\b(?!000|666|9\d\d)\d{3}-(?!00)\d{2}-(?!0000)\d{4}\b"),
+        "[REDACTED:SSN]",
+    ),
+    (
+        re.compile(
+            r"\b(?:password|passwd|pwd)\s*[:=]\s*(['\"]?)([^\s'\",;]{8,})\1",
+            re.I,
+        ),
+        "[REDACTED:PASSWORD]",
+    ),
+    (
+        re.compile(
+            r"\b(?:api[_-]?key|access[_-]?key|secret[_-]?key|client[_-]?secret)\s*[:=]\s*(['\"]?)([A-Za-z0-9._-]{20,})\1",
+            re.I,
+        ),
+        "[REDACTED:API_KEY]",
+    ),
 ]
 
 
@@ -49,8 +76,13 @@ def ts_from_input(value: str | None) -> str | None:
 
 def redact(text: str) -> str:
     out = text
-    for pattern in TOKEN_PATTERNS:
-        out = pattern.sub("[REDACTED]", out)
+    for pattern, replacement in TOKEN_PATTERNS:
+        out = pattern.sub(
+            lambda match: match.group(0).replace(match.group(2), replacement)
+            if len(match.groups()) >= 2
+            else replacement,
+            out,
+        )
     return out
 
 
@@ -289,7 +321,7 @@ def main() -> int:
             for reply in replies:
                 lines.extend(render_message(reply))
 
-    source_path.write_text("\n".join(lines), encoding="utf-8")
+    source_path.write_text(redact("\n".join(lines)), encoding="utf-8")
 
     latest_ts = previous_state.get("latest_message_ts")
     if messages: