@lanonasis/recall-forge 1.1.1 → 1.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/dist/extraction/index.d.ts +1 -0
- package/dist/extraction/jsonl-extractor.js +1 -1
- package/dist/extraction/markdown-extractor.js +1 -1
- package/dist/extraction/secret-redactor.d.ts +5 -5
- package/dist/extraction/secret-redactor.js +105 -66
- package/dist/extraction/sqlite-extractor.js +1 -1
- package/package.json +2 -2
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,21 @@
|
|
|
2
2
|
|
|
3
3
|
All notable changes to `@lanonasis/recall-forge` will be documented in this file.
|
|
4
4
|
|
|
5
|
+
## [Unreleased]
|
|
6
|
+
|
|
7
|
+
---
|
|
8
|
+
|
|
9
|
+
## 1.1.2 — 2026-09-23
|
|
10
|
+
|
|
11
|
+
### Fixed
|
|
12
|
+
|
|
13
|
+
- **Repository URL corrected.** `repository.url` no longer points at the private monorepo (`thefixer3x/lan-onasis-monorepo.git`); it is now `git+https://github.com/lanonasis/lanonasis-maas.git` with `directory: packages/recall-forge`, so npm and GitHub Packages link to the public source.
|
|
14
|
+
- **Secret redactor realigned with 1.1.1.** The source copy of `extraction/secret-redactor.ts` had been hidden by a `.gitignore` `*key*` rule, and the restored file dropped rules that shipped in 1.1.1. It now keeps the newer provider rules (GitHub, Stripe, Google, Notion, ElevenLabs, Telegram, `sb_` tokens, PEM private keys, 64+ hex secrets) and restores the 1.1.1 ones: lowercase / `key: value` assignments for `api_key`, `password`, `secret_key`/`private_key` and `aws_secret_access_key`, plus PII (email, credit card, SSN, phone).
|
|
15
|
+
- `redactSecrets(text, { redactPII })` and `containsSecrets(text, { redactPII })` accept options again (PII on by default, as in 1.1.1), and the JSONL, Markdown and SQLite extractors now honour `ExtractionOptions.redactPII`.
|
|
16
|
+
- OpenAI key detection matches 20+ character keys again (was 32+).
|
|
17
|
+
- `containsSecrets()` no longer depends on global-regex `lastIndex` state, so repeated calls give the same answer.
|
|
18
|
+
- The 1.1.1 catch-all `supabase-key` rule (any 40+ character token) is not restored: it redacted git SHAs and ids. Supabase JWTs and `sb_` keys are covered by `jwt-token` and `supabase-token`.
|
|
19
|
+
|
|
5
20
|
---
|
|
6
21
|
|
|
7
22
|
## 1.1.1 — 2026-04-04
|
|
@@ -2,6 +2,7 @@ export { extractJsonl, formatStats } from "./jsonl-extractor.js";
|
|
|
2
2
|
export { extractMarkdown, isMarkdownFile } from "./markdown-extractor.js";
|
|
3
3
|
export { extractSqlite, isSqliteFile } from "./sqlite-extractor.js";
|
|
4
4
|
export { redactSecrets, containsSecrets } from "./secret-redactor.js";
|
|
5
|
+
export type { RedactOptions } from "./secret-redactor.js";
|
|
5
6
|
export { detectFormat, FORMAT_ADAPTERS } from "./format-adapters.js";
|
|
6
7
|
export { registerExtractCli } from "./cli-extract.js";
|
|
7
8
|
export type { ExtractionOptions, ExtractionStats, ExtractionRecord, RedactionResult, FormatAdapter, } from "./types.js";
|
|
@@ -88,7 +88,7 @@ export async function extractJsonl(options, deps) {
|
|
|
88
88
|
if (roles.length > 0 && !roles.includes(record.role))
|
|
89
89
|
continue;
|
|
90
90
|
// Step 1: Redact secrets FIRST — before any other processing
|
|
91
|
-
const redaction = redactSecrets(record.text);
|
|
91
|
+
const redaction = redactSecrets(record.text, { redactPII: options.redactPII });
|
|
92
92
|
stats.secretsRedacted += redaction.secretsFound;
|
|
93
93
|
const cleanText = redaction.text;
|
|
94
94
|
// Step 2: Capture filter
|
|
@@ -142,7 +142,7 @@ export async function extractMarkdown(options, deps) {
|
|
|
142
142
|
if (roles.length > 0 && !roles.includes(record.role))
|
|
143
143
|
continue;
|
|
144
144
|
// Step 1: Redact secrets
|
|
145
|
-
const redaction = redactSecrets(record.text);
|
|
145
|
+
const redaction = redactSecrets(record.text, { redactPII: options.redactPII });
|
|
146
146
|
stats.secretsRedacted += redaction.secretsFound;
|
|
147
147
|
const cleanText = redaction.text;
|
|
148
148
|
// Step 2: Capture filter
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { RedactionResult } from "./types.js";
|
|
2
|
-
export
|
|
2
|
+
export interface RedactOptions {
|
|
3
|
+
/** Also redact PII (email, credit card, SSN, phone). Default: true. */
|
|
3
4
|
redactPII?: boolean;
|
|
4
|
-
}
|
|
5
|
-
export declare function
|
|
6
|
-
|
|
7
|
-
}): boolean;
|
|
5
|
+
}
|
|
6
|
+
export declare function redactSecrets(input: string, options?: RedactOptions): RedactionResult;
|
|
7
|
+
export declare function containsSecrets(input: string, options?: RedactOptions): boolean;
|
|
@@ -1,112 +1,151 @@
|
|
|
1
|
-
// Secret redaction for JSONL extraction
|
|
2
|
-
// Detects and redacts API keys, tokens, credentials, and PII
|
|
3
1
|
const SECRET_PATTERNS = [
|
|
4
2
|
{
|
|
5
|
-
|
|
6
|
-
pattern: /\
|
|
3
|
+
type: "anthropic-api-key",
|
|
4
|
+
pattern: /\bsk-ant-[A-Za-z0-9_-]{20,}\b/g,
|
|
7
5
|
},
|
|
8
6
|
{
|
|
9
|
-
|
|
10
|
-
pattern: /\bsk-[A-Za-z0-
|
|
7
|
+
type: "openai-api-key",
|
|
8
|
+
pattern: /\bsk-(?:proj-)?[A-Za-z0-9_-]{20,}\b/g,
|
|
11
9
|
},
|
|
12
10
|
{
|
|
13
|
-
|
|
14
|
-
pattern: /\
|
|
11
|
+
type: "github-token",
|
|
12
|
+
pattern: /\b(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9_]{20,}\b/g,
|
|
15
13
|
},
|
|
16
14
|
{
|
|
17
|
-
|
|
18
|
-
pattern: /\
|
|
15
|
+
type: "github-token",
|
|
16
|
+
pattern: /\bgithub_pat_[A-Za-z0-9_]{20,}\b/g,
|
|
19
17
|
},
|
|
20
18
|
{
|
|
21
|
-
|
|
22
|
-
pattern: /\
|
|
19
|
+
type: "supabase-token",
|
|
20
|
+
pattern: /\bsb[ap]_[A-Za-z0-9_-]{20,}\b/g,
|
|
23
21
|
},
|
|
24
22
|
{
|
|
25
|
-
|
|
26
|
-
pattern: /\
|
|
23
|
+
type: "stripe-key",
|
|
24
|
+
pattern: /\b(?:sk|pk)_(?:live|test)_[A-Za-z0-9]{20,}\b/g,
|
|
27
25
|
},
|
|
28
26
|
{
|
|
29
|
-
|
|
30
|
-
pattern: /\
|
|
27
|
+
type: "stripe-webhook-secret",
|
|
28
|
+
pattern: /\bwhsec_[A-Za-z0-9]{20,}\b/g,
|
|
31
29
|
},
|
|
32
30
|
{
|
|
33
|
-
|
|
34
|
-
pattern: /\
|
|
31
|
+
type: "aws-access-key",
|
|
32
|
+
pattern: /\b(?:AKIA|ASIA)[A-Z0-9]{16}\b/g,
|
|
33
|
+
},
|
|
34
|
+
{
|
|
35
|
+
type: "google-api-key",
|
|
36
|
+
pattern: /\bAIza[0-9A-Za-z_-]{30,}\b/g,
|
|
35
37
|
},
|
|
36
38
|
{
|
|
37
|
-
|
|
38
|
-
pattern: /\b(
|
|
39
|
+
type: "notion-token",
|
|
40
|
+
pattern: /\b(?:ntn|secret)_[A-Za-z0-9]{20,}\b/g,
|
|
39
41
|
},
|
|
40
42
|
{
|
|
41
|
-
|
|
42
|
-
pattern: /\b(
|
|
43
|
+
type: "lanonasis-api-key",
|
|
44
|
+
pattern: /\b(?:lano|lns)_[A-Za-z0-9_-]{20,}\b/g,
|
|
43
45
|
},
|
|
44
46
|
{
|
|
45
|
-
|
|
46
|
-
pattern: /\
|
|
47
|
+
type: "jwt-token",
|
|
48
|
+
pattern: /\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b/g,
|
|
47
49
|
},
|
|
48
50
|
{
|
|
49
|
-
|
|
50
|
-
pattern: /\
|
|
51
|
+
type: "bearer-token",
|
|
52
|
+
pattern: /\bBearer\s+[A-Za-z0-9._~+/=-]{20,}\b/g,
|
|
53
|
+
},
|
|
54
|
+
{
|
|
55
|
+
type: "database-url",
|
|
56
|
+
pattern: /\b(?:postgres(?:ql)?|mysql|mongodb(?:\+srv)?|redis):\/\/[^\s"'<>]+/gi,
|
|
57
|
+
},
|
|
58
|
+
{
|
|
59
|
+
type: "private-key",
|
|
60
|
+
pattern: /-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/g,
|
|
61
|
+
},
|
|
62
|
+
{
|
|
63
|
+
type: "hex-secret",
|
|
64
|
+
pattern: /\b[a-f0-9]{64,}\b/gi,
|
|
65
|
+
},
|
|
66
|
+
{
|
|
67
|
+
type: "elevenlabs-api-key",
|
|
68
|
+
pattern: /\bel_[A-Za-z0-9_-]{20,}\b/g,
|
|
69
|
+
},
|
|
70
|
+
{
|
|
71
|
+
type: "telegram-bot-token",
|
|
72
|
+
pattern: /\b\d{8,10}:[A-Za-z0-9_-]{30,}\b/g,
|
|
51
73
|
},
|
|
52
74
|
];
|
|
53
|
-
|
|
75
|
+
// Lowercase / `key: value` credential assignments that ASSIGNMENT_PATTERN
|
|
76
|
+
// (UPPER_CASE env style only) misses. Carried over from the 1.1.1 redactor.
|
|
77
|
+
const KEYED_SECRET_PATTERNS = [
|
|
54
78
|
{
|
|
55
|
-
|
|
56
|
-
pattern: /\
|
|
79
|
+
type: "aws-secret-key",
|
|
80
|
+
pattern: /\baws[_-]?secret[_-]?access[_-]?key\s*[:=]\s*["']?[A-Za-z0-9/+=]{40}["']?/gi,
|
|
57
81
|
},
|
|
58
82
|
{
|
|
59
|
-
|
|
60
|
-
pattern:
|
|
83
|
+
type: "generic-api-key",
|
|
84
|
+
pattern: /\b(?:api[_-]?key|apikey)\s*[:=]\s*["']?[A-Za-z0-9_-]{16,}["']?/gi,
|
|
85
|
+
},
|
|
86
|
+
{
|
|
87
|
+
type: "secret-key",
|
|
88
|
+
pattern: /\b(?:secret[_-]?key|private[_-]?key)\s*[:=]\s*["']?[A-Za-z0-9_-]{16,}["']?/gi,
|
|
89
|
+
},
|
|
90
|
+
{
|
|
91
|
+
type: "password",
|
|
92
|
+
pattern: /\b(?:password|passwd|pwd)\s*[:=]\s*["']?[^\s"']{8,}["']?/gi,
|
|
93
|
+
},
|
|
94
|
+
];
|
|
95
|
+
// PII, on by default (1.1.1 behaviour). Callers that run their own PII stage
|
|
96
|
+
// can pass { redactPII: false }.
|
|
97
|
+
const PII_PATTERNS = [
|
|
98
|
+
{
|
|
99
|
+
type: "email",
|
|
100
|
+
pattern: /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b/g,
|
|
61
101
|
},
|
|
62
102
|
{
|
|
63
|
-
|
|
103
|
+
type: "credit-card",
|
|
64
104
|
pattern: /\b\d{4}[\s-]?\d{4}[\s-]?\d{4}[\s-]?\d{4}\b/g,
|
|
65
105
|
},
|
|
66
106
|
{
|
|
67
|
-
|
|
107
|
+
type: "ssn",
|
|
68
108
|
pattern: /\b\d{3}[\s-]?\d{2}[\s-]?\d{4}\b/g,
|
|
69
109
|
},
|
|
110
|
+
{
|
|
111
|
+
type: "phone",
|
|
112
|
+
pattern: /\+?[\d\s()-]{10,}\b/g,
|
|
113
|
+
},
|
|
70
114
|
];
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
const secretsFound = [];
|
|
115
|
+
const ASSIGNMENT_PATTERN = /\b((?:export\s+)?[A-Z][A-Z0-9_]*(?:API[_-]?KEY|TOKEN|SECRET|PASSWORD|PRIVATE[_-]?KEY|ACCESS[_-]?TOKEN|REFRESH[_-]?TOKEN)[A-Z0-9_]*\s*=\s*)(["']?)([^\s"'`]+)(\2)/gi;
|
|
116
|
+
export function redactSecrets(input, options = {}) {
|
|
74
117
|
const { redactPII = true } = options;
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
118
|
+
let text = input;
|
|
119
|
+
const types = [];
|
|
120
|
+
let secretsFound = 0;
|
|
121
|
+
const mark = (type) => {
|
|
122
|
+
secretsFound++;
|
|
123
|
+
if (!types.includes(type))
|
|
124
|
+
types.push(type);
|
|
125
|
+
return `[REDACTED:${type}]`;
|
|
126
|
+
};
|
|
127
|
+
for (const { type, pattern } of SECRET_PATTERNS) {
|
|
128
|
+
text = text.replace(pattern, () => mark(type));
|
|
129
|
+
}
|
|
130
|
+
text = text.replace(ASSIGNMENT_PATTERN, (_match, prefix, quote, value, closingQuote) => {
|
|
131
|
+
if (value.startsWith("[REDACTED:"))
|
|
132
|
+
return `${prefix}${quote}${value}${closingQuote}`;
|
|
133
|
+
return `${prefix}${quote}${mark("env-secret")}${closingQuote}`;
|
|
134
|
+
});
|
|
135
|
+
for (const { type, pattern } of KEYED_SECRET_PATTERNS) {
|
|
136
|
+
text = text.replace(pattern, (match) => (match.includes("[REDACTED:") ? match : mark(type)));
|
|
81
137
|
}
|
|
82
138
|
if (redactPII) {
|
|
83
|
-
for (const {
|
|
84
|
-
|
|
85
|
-
if (matches && matches.length > 0) {
|
|
86
|
-
secretsFound.push(name);
|
|
87
|
-
result = result.replace(pattern, `[REDACTED:${name}]`);
|
|
88
|
-
}
|
|
139
|
+
for (const { type, pattern } of PII_PATTERNS) {
|
|
140
|
+
text = text.replace(pattern, () => mark(type));
|
|
89
141
|
}
|
|
90
142
|
}
|
|
91
143
|
return {
|
|
92
|
-
text
|
|
93
|
-
secretsFound
|
|
94
|
-
types
|
|
144
|
+
text,
|
|
145
|
+
secretsFound,
|
|
146
|
+
types,
|
|
95
147
|
};
|
|
96
148
|
}
|
|
97
|
-
export function containsSecrets(
|
|
98
|
-
|
|
99
|
-
for (const { pattern } of SECRET_PATTERNS) {
|
|
100
|
-
if (pattern.test(text)) {
|
|
101
|
-
return true;
|
|
102
|
-
}
|
|
103
|
-
}
|
|
104
|
-
if (redactPII) {
|
|
105
|
-
for (const { pattern } of PII_PATTERNS) {
|
|
106
|
-
if (pattern.test(text)) {
|
|
107
|
-
return true;
|
|
108
|
-
}
|
|
109
|
-
}
|
|
110
|
-
}
|
|
111
|
-
return false;
|
|
149
|
+
export function containsSecrets(input, options = {}) {
|
|
150
|
+
return redactSecrets(input, options).secretsFound > 0;
|
|
112
151
|
}
|
|
@@ -149,7 +149,7 @@ export async function extractSqlite(options, deps) {
|
|
|
149
149
|
continue;
|
|
150
150
|
}
|
|
151
151
|
// Step 1: Redact secrets
|
|
152
|
-
const redaction = redactSecrets(chunk.text);
|
|
152
|
+
const redaction = redactSecrets(chunk.text, { redactPII: options.redactPII });
|
|
153
153
|
stats.secretsRedacted += redaction.secretsFound;
|
|
154
154
|
const cleanText = redaction.text;
|
|
155
155
|
// Step 2: Capture filter
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lanonasis/recall-forge",
|
|
3
|
-
"version": "1.1.
|
|
3
|
+
"version": "1.1.2",
|
|
4
4
|
"description": "RecallForge — secret-safe memory and context engine plugin for OpenClaw. Fills both memory and contextEngine slots with tiered semantic recall, 30+ pattern credential redaction, and prompt injection protection.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -52,7 +52,7 @@
|
|
|
52
52
|
},
|
|
53
53
|
"repository": {
|
|
54
54
|
"type": "git",
|
|
55
|
-
"url": "https://github.com/
|
|
55
|
+
"url": "git+https://github.com/lanonasis/lanonasis-maas.git",
|
|
56
56
|
"directory": "packages/recall-forge"
|
|
57
57
|
},
|
|
58
58
|
"homepage": "https://github.com/lanonasis/lanonasis-maas/tree/main/packages/recall-forge#readme",
|