@blanklogic/refinery-core 0.1.0

package/README.md

@@ -0,0 +1,21 @@
+# BlankLogic Refinery Core
+
+Shared local engine package for BlankLogic Refinery.
+
+```bash
+npm install @blanklogic/refinery-core
+```
+
+## API
+
+```js
+import { refineFull, runCodeDebloater, runTokenSanitizer } from '@blanklogic/refinery-core'
+
+const cleaned = refineFull(input)
+```
+
+- `runCodeDebloater(input)` removes tracker noise, comments, and boilerplate while preserving useful structure.
+- `runTokenSanitizer(input)` redacts obvious secrets and compacts text for LLM context windows.
+- `refineFull(input)` runs de-bloat first, then secret sanitization.
+
+This package is local-only. It does not call Stripe, BlankLogic servers, Ollama, or third-party APIs.

package/package.json

@@ -0,0 +1,32 @@
+{
+  "name": "@blanklogic/refinery-core",
+  "version": "0.1.0",
+  "description": "Shared BlankLogic Refinery context hygiene engines.",
+  "type": "module",
+  "private": false,
+  "exports": {
+    ".": "./src/index.js"
+  },
+  "files": [
+    "src"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "license": "UNLICENSED",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/briancrabtree-me/blanklogic-site.git",
+    "directory": "packages/core"
+  },
+  "keywords": [
+    "blanklogic",
+    "refinery",
+    "llm",
+    "context",
+    "sanitizer"
+  ],
+  "engines": {
+    "node": ">=20"
+  }
+}

package/src/index.js

@@ -0,0 +1,36 @@
+import { runCodeDebloater, roughTokenCount, byteSize, lineCount } from './refineryCompiler.js'
+import { runTokenSanitizer } from './refinerySanitizer.js'
+
+export { runCodeDebloater, roughTokenCount, byteSize, lineCount, runTokenSanitizer }
+
+export function refineFull(inputValue) {
+  const input = String(inputValue ?? '')
+  const debloated = runCodeDebloater(input)
+  const sanitized = runTokenSanitizer(debloated.output)
+  const bytesIn = byteSize(input)
+  const bytesOut = byteSize(sanitized.output)
+  const tokensBefore = roughTokenCount(input)
+  const tokensAfter = roughTokenCount(sanitized.output)
+
+  return {
+    output: sanitized.output,
+    stats: {
+      bytesIn,
+      bytesOut,
+      tokensBefore,
+      tokensAfter,
+      tokensSaved: Math.max(tokensBefore - tokensAfter, 0),
+      linesBefore: lineCount(input),
+      linesAfter: lineCount(sanitized.output),
+      reduction: bytesIn > 0 ? Math.max(0, Math.round((1 - bytesOut / bytesIn) * 100)) : 0,
+      trackersRemoved: debloated.trackersRemoved,
+      htmlCommentsRemoved: debloated.htmlCommentsRemoved,
+      scriptCommentsRemoved: debloated.scriptCommentsRemoved,
+      styleCommentsRemoved: debloated.styleCommentsRemoved,
+      secretsRedacted: sanitized.secretsRedacted,
+      commentsRemoved: sanitized.commentsRemoved,
+      domSafe: debloated.domSafe,
+      ms: debloated.ms + sanitized.ms,
+    },
+  }
+}

package/src/refineryCompiler.js

@@ -0,0 +1,263 @@
+const TRACKER_BLOCK_RE = /<(script|noscript)\b[^>]*>[\s\S]*?<\/\1>/gi
+
+const TRACKER_VENDOR_RE =
+  /google-analytics|googletagmanager|gtag|hotjar|segment|mixpanel|facebook|fbq|clarity/i
+
+const RAW_BLOCK_RE = /<(script|style|pre|code|textarea)\b[^>]*>[\s\S]*?<\/\1>/gi
+
+export function runCodeDebloater(inputValue) {
+  const start = now()
+  const input = String(inputValue ?? '')
+  const normalized = input.replace(/\r\n?/g, '\n')
+  const htmlLike = /<\/?[a-z][\s\S]*>/i.test(normalized)
+  const result = htmlLike ? compactHtml(normalized) : compactLooseCode(normalized)
+  const output = result.output.trim()
+
+  return {
+    output,
+    ms: now() - start,
+    reduction: reductionPercent(input, output),
+    tokensSaved: Math.max(roughTokenCount(input) - roughTokenCount(output), 0),
+    bytesIn: byteSize(input),
+    bytesOut: byteSize(output),
+    linesBefore: lineCount(input),
+    linesAfter: lineCount(output),
+    trackersRemoved: result.stats.trackersRemoved,
+    htmlCommentsRemoved: result.stats.htmlCommentsRemoved,
+    scriptCommentsRemoved: result.stats.scriptCommentsRemoved,
+    styleCommentsRemoved: result.stats.styleCommentsRemoved,
+    domSafe: htmlLike,
+  }
+}
+
+function compactHtml(input) {
+  const stats = emptyStats()
+  let working = input.replace(TRACKER_BLOCK_RE, (block) => {
+    if (!TRACKER_VENDOR_RE.test(block)) return block
+    stats.trackersRemoved += 1
+    return ''
+  })
+
+  const blocks = []
+  working = working.replace(RAW_BLOCK_RE, (block, tagName) => {
+    const tag = String(tagName).toLowerCase()
+    let next = block
+    if (tag === 'script') {
+      const stripped = processScriptBlock(block)
+      stats.scriptCommentsRemoved += stripped.commentsRemoved
+      next = stripped.output
+    } else if (tag === 'style') {
+      const stripped = processStyleBlock(block)
+      stats.styleCommentsRemoved += stripped.commentsRemoved
+      next = stripped.output
+    }
+    const marker = `\u0000REFINERY_BLOCK_${blocks.length}\u0000`
+    blocks.push(next)
+    return marker
+  })
+
+  working = working.replace(/<!--[\s\S]*?-->/g, () => {
+    stats.htmlCommentsRemoved += 1
+    return ''
+  })
+
+  working = working
+    .split(/(<[^>]+>|\u0000REFINERY_BLOCK_\d+\u0000)/g)
+    .map((part) => {
+      if (!part) return ''
+      if (part.startsWith('\u0000REFINERY_BLOCK_')) return part
+      if (part.startsWith('<')) return part.replace(/\s{2,}/g, ' ').trim()
+      return compactTextNode(part)
+    })
+    .join('')
+    .replace(/>\s+</g, '><')
+    .replace(/\n{3,}/g, '\n\n')
+
+  const output = working.replace(/\u0000REFINERY_BLOCK_(\d+)\u0000/g, (_m, i) => blocks[Number(i)] || '')
+  return { output, stats }
+}
+
+function compactLooseCode(input) {
+  const stats = emptyStats()
+  const withoutHtmlComments = input.replace(/<!--[\s\S]*?-->/g, () => {
+    stats.htmlCommentsRemoved += 1
+    return ''
+  })
+  const script = stripJsComments(withoutHtmlComments)
+  stats.scriptCommentsRemoved = script.commentsRemoved
+  const style = stripCssComments(script.output)
+  stats.styleCommentsRemoved = style.commentsRemoved
+  return {
+    output: style.output
+      .replace(/[ \t]{2,}/g, ' ')
+      .replace(/\n{3,}/g, '\n\n')
+      .trim(),
+    stats,
+  }
+}
+
+function processScriptBlock(block) {
+  const parts = splitRawBlock(block)
+  const stripped = stripJsComments(parts.body)
+  return {
+    output: `${parts.open}${stripped.output.trim()}${parts.close}`,
+    commentsRemoved: stripped.commentsRemoved,
+  }
+}
+
+function processStyleBlock(block) {
+  const parts = splitRawBlock(block)
+  const stripped = stripCssComments(parts.body)
+  return {
+    output: `${parts.open}${stripped.output.trim()}${parts.close}`,
+    commentsRemoved: stripped.commentsRemoved,
+  }
+}
+
+function splitRawBlock(block) {
+  const openMatch = block.match(/^<[^>]+>/)
+  const closeMatch = block.match(/<\/[a-z]+>\s*$/i)
+  const open = openMatch?.[0] || ''
+  const close = closeMatch?.[0] || ''
+  const body = block.slice(open.length, block.length - close.length)
+  return { open, body, close }
+}
+
+function stripCssComments(input) {
+  let output = ''
+  let commentsRemoved = 0
+  let quote = ''
+  let escaped = false
+
+  for (let i = 0; i < input.length; i += 1) {
+    const ch = input[i]
+    const next = input[i + 1]
+
+    if (quote) {
+      output += ch
+      if (escaped) {
+        escaped = false
+      } else if (ch === '\\') {
+        escaped = true
+      } else if (ch === quote) {
+        quote = ''
+      }
+      continue
+    }
+
+    if (ch === '"' || ch === "'") {
+      quote = ch
+      output += ch
+      continue
+    }
+
+    if (ch === '/' && next === '*') {
+      commentsRemoved += 1
+      i += 2
+      while (i < input.length && !(input[i] === '*' && input[i + 1] === '/')) i += 1
+      i += 1
+      continue
+    }
+
+    output += ch
+  }
+
+  return { output, commentsRemoved }
+}
+
+function stripJsComments(input) {
+  let output = ''
+  let commentsRemoved = 0
+  let quote = ''
+  let escaped = false
+
+  for (let i = 0; i < input.length; i += 1) {
+    const ch = input[i]
+    const next = input[i + 1]
+
+    if (quote) {
+      output += ch
+      if (escaped) {
+        escaped = false
+      } else if (ch === '\\') {
+        escaped = true
+      } else if (ch === quote) {
+        quote = ''
+      }
+      continue
+    }
+
+    if (ch === '"' || ch === "'" || ch === '`') {
+      quote = ch
+      output += ch
+      continue
+    }
+
+    if (ch === '/' && next === '/' && isSlashCommentStart(output)) {
+      commentsRemoved += 1
+      while (i < input.length && input[i] !== '\n' && input[i] !== '\r') i += 1
+      output += input[i] || ''
+      continue
+    }
+
+    if (ch === '/' && next === '*') {
+      commentsRemoved += 1
+      i += 2
+      while (i < input.length && !(input[i] === '*' && input[i + 1] === '/')) i += 1
+      i += 1
+      continue
+    }
+
+    output += ch
+  }
+
+  return { output, commentsRemoved }
+}
+
+function isSlashCommentStart(output) {
+  const trimmed = output.replace(/[ \t]+$/g, '')
+  const prev = trimmed[trimmed.length - 1] || ''
+  return !prev || prev === '\n' || prev === '\r' || prev === ';' || prev === '{' || prev === '}'
+}
+
+function compactTextNode(text) {
+  return text
+    .replace(/[ \t]{2,}/g, ' ')
+    .replace(/\s*\n\s*/g, ' ')
+}
+
+function emptyStats() {
+  return {
+    trackersRemoved: 0,
+    htmlCommentsRemoved: 0,
+    scriptCommentsRemoved: 0,
+    styleCommentsRemoved: 0,
+  }
+}
+
+export function roughTokenCount(value) {
+  return String(value)
+    .trim()
+    .split(/[\s{}[\]():;,.'"`<>/\\|+=*-]+/)
+    .filter(Boolean).length
+}
+
+export function byteSize(value) {
+  if (typeof TextEncoder !== 'undefined') return new TextEncoder().encode(String(value)).length
+  return Buffer.byteLength(String(value), 'utf8')
+}
+
+export function lineCount(value) {
+  const text = String(value || '')
+  return text.length ? text.replace(/\r\n?/g, '\n').split('\n').length : 0
+}
+
+function reductionPercent(input, output) {
+  const before = byteSize(input)
+  const after = byteSize(output)
+  return before > 0 ? Math.max(0, Math.round((1 - after / before) * 100)) : 0
+}
+
+function now() {
+  return typeof performance !== 'undefined' && performance.now ? performance.now() : Date.now()
+}

package/src/refinerySanitizer.js

@@ -0,0 +1,171 @@
+const SECRET_PATTERNS = [
+  {
+    pattern: /\bsk-[A-Za-z0-9_-]{12,}\b/g,
+    replacement: '[REDACTED_STRIPE_SECRET]',
+  },
+  {
+    pattern: /\bpk_(live|test)_[A-Za-z0-9]{12,}\b/g,
+    replacement: '[REDACTED_STRIPE_PUBLIC]',
+  },
+  {
+    pattern: /\bAIza[0-9A-Za-z_-]{20,}\b/g,
+    replacement: '[REDACTED_GOOGLE_API_KEY]',
+  },
+  {
+    pattern: /\b(?:ghp|github_pat|gho|ghu|ghs)_[A-Za-z0-9_]{20,}\b/g,
+    replacement: '[REDACTED_GITHUB_TOKEN]',
+  },
+  {
+    pattern: /\b[A-Za-z0-9_-]{24,}\.[A-Za-z0-9_-]{12,}\.[A-Za-z0-9_-]{20,}\b/g,
+    replacement: '[REDACTED_JWT]',
+  },
+  {
+    pattern: /\b(Authorization\s*:\s*Bearer\s+)(?!\[REDACTED)[^\s,;&]+/gi,
+    replacement: '$1[REDACTED_BEARER]',
+  },
+  {
+    pattern: /\b([A-Za-z0-9_-]*(?:api[_-]?key|token|secret|password)[A-Za-z0-9_-]*)\s*[:=]\s*["']?(?!\[REDACTED)[^"'\s,;&]+["']?/gi,
+    replacement: '$1=[REDACTED]',
+  },
+]
+
+const NAMED_SECRET_ASSIGNMENT_RE =
+  /(^|[{\s,;])(["']?[A-Za-z0-9_-]*(?:api[_-]?key|token|secret|password)[A-Za-z0-9_-]*["']?\s*[:=]\s*)(["']?)(?!\[REDACTED\])([^"'\s,;&}]+)(["']?)/gi
+
+export function runTokenSanitizer(inputValue) {
+  const start = now()
+  const input = String(inputValue ?? '')
+  const normalized = input.replace(/\r\n?/g, '\n')
+  let secretsRedacted = 0
+  let protectedText = normalized.replace(NAMED_SECRET_ASSIGNMENT_RE, (_match, prefix, assignment, quote, _value) => {
+    secretsRedacted += 1
+    return `${prefix}${assignment}${quote}[REDACTED]${quote}`
+  })
+  for (const item of SECRET_PATTERNS) {
+    protectedText = protectedText.replace(item.pattern, (match) => {
+      secretsRedacted += 1
+      return replaceSingle(match, item.pattern, item.replacement)
+    })
+  }
+
+  let commentsRemoved = 0
+  let working = protectedText
+    .replace(/("""|''')[\s\S]*?\1/g, () => {
+      commentsRemoved += 1
+      return ''
+    })
+    .replace(/<!--[\s\S]*?-->/g, () => {
+      commentsRemoved += 1
+      return ''
+    })
+    .replace(/(^|\n)[ \t]*#(?![A-Fa-f0-9]{3,8}\b)[^\n\r]*/g, (match, prefix) => {
+      commentsRemoved += 1
+      return prefix
+    })
+
+  const slash = stripSlashComments(working)
+  commentsRemoved += slash.commentsRemoved
+  working = slash.output
+
+  const output = working
+    .replace(/```[\s\S]*?```/g, (block) => block.replace(/\n{2,}/g, '\n'))
+    .split('\n')
+    .map((line) => line.trim())
+    .filter(Boolean)
+    .join(' ')
+    .replace(/\s{2,}/g, ' ')
+    .replace(/\s*([{}[\]():;,=+\-*<>|])\s*/g, '$1')
+    .trim()
+
+  return {
+    output,
+    ms: now() - start,
+    secretsRedacted,
+    commentsRemoved,
+    tokensSaved: Math.max(roughTokenCount(input) - roughTokenCount(output), 0),
+    bytesIn: byteSize(input),
+    bytesOut: byteSize(output),
+  }
+}
+
+function roughTokenCount(value) {
+  return String(value)
+    .trim()
+    .split(/[\s{}[\]():;,.'"`<>/\\|+=*-]+/)
+    .filter(Boolean).length
+}
+
+function byteSize(value) {
+  if (typeof TextEncoder !== 'undefined') return new TextEncoder().encode(String(value)).length
+  return Buffer.byteLength(String(value), 'utf8')
+}
+
+function replaceSingle(value, pattern, replacement) {
+  const flags = pattern.flags.replace('g', '')
+  return String(value || '').replace(new RegExp(pattern.source, flags), replacement)
+}
+
+function stripSlashComments(input) {
+  let output = ''
+  let commentsRemoved = 0
+  let quote = ''
+  let escaped = false
+
+  for (let i = 0; i < input.length; i += 1) {
+    const ch = input[i]
+    const next = input[i + 1]
+
+    if (quote) {
+      output += ch
+      if (escaped) {
+        escaped = false
+      } else if (ch === '\\') {
+        escaped = true
+      } else if (ch === quote) {
+        quote = ''
+      }
+      continue
+    }
+
+    if (ch === '`' && next === '`' && input[i + 2] === '`') {
+      output += '```'
+      i += 2
+      continue
+    }
+
+    if (ch === '"' || ch === "'" || ch === '`') {
+      quote = ch
+      output += ch
+      continue
+    }
+
+    if (ch === '/' && next === '/' && isLineCommentStart(output)) {
+      commentsRemoved += 1
+      while (i < input.length && input[i] !== '\n' && input[i] !== '\r') i += 1
+      output += input[i] || ''
+      continue
+    }
+
+    if (ch === '/' && next === '*') {
+      commentsRemoved += 1
+      i += 2
+      while (i < input.length && !(input[i] === '*' && input[i + 1] === '/')) i += 1
+      i += 1
+      continue
+    }
+
+    output += ch
+  }
+
+  return { output, commentsRemoved }
+}
+
+function isLineCommentStart(output) {
+  const trimmed = output.replace(/[ \t]+$/g, '')
+  const prev = trimmed[trimmed.length - 1] || ''
+  return !prev || prev === '\n' || prev === '\r' || prev === ';' || prev === '{' || prev === '}'
+}
+
+function now() {
+  return typeof performance !== 'undefined' && performance.now ? performance.now() : Date.now()
+}