@redpanda-data/docs-extensions-and-macros 4.2.5 → 4.4.0

package/docker-compose/transform/transform.go

@@ -0,0 +1,122 @@
+package main
+// This data transform filters records based on a customizable regex pattern.
+// If a record's key or value
+// (determined by an environment variable) matches the specified regex,
+// the record is forwarded to the output.
+// Otherwise, it is dropped.
+//
+// Usage:
+// 1. Provide the following environment variables in your Docker or configuration setup:
+//    - PATTERN     : (required) a regular expression that determines what you want to match.
+//    - MATCH_VALUE : (optional) a boolean to decide whether to check the record value. If false,
+//                   the record key is checked. Default is false.
+//
+// Example environment variables:
+//   PATTERN=".*\\.edu$"
+//   MATCH_VALUE="true"
+//
+// Logs:
+// This transform logs information about each record and whether it matched.
+// The logs appear in the _redpanda.transform_logs topic, so you can debug how your records are being processed.
+//
+// Build instructions:
+//   go mod tidy
+//   rpk transform build
+//
+// For more details on building transforms with the Redpanda SDK, see:
+// https://docs.redpanda.com/current/develop/data-transforms
+//
+
+import (
+	"log"
+	"os"
+	"regexp"
+	"strings"
+
+	"github.com/redpanda-data/redpanda/src/transform-sdk/go/transform"
+)
+
+var (
+	re         *regexp.Regexp
+	checkValue bool
+)
+
+func isTrueVar(v string) bool {
+	switch strings.ToLower(v) {
+	case "yes", "ok", "1", "true":
+		return true
+	default:
+		return false
+	}
+}
+
+// The main() function runs only once at startup. It performs all initialization steps:
+// - Reads and compiles the regex pattern.
+// - Determines whether to match on the key or value.
+// - Registers the doRegexFilter() function to process records.
+func main() {
+	// Set logging preferences, including timestamp and UTC time.
+	log.SetPrefix("[regex-transform] ")
+	log.SetFlags(log.Ldate | log.Ltime | log.LUTC | log.Lmicroseconds)
+
+	// Start logging the transformation process
+	log.Println("Starting transform...")
+
+	// Read the PATTERN environment variable to get the regex pattern.
+	pattern, ok := os.LookupEnv("PATTERN")
+	if !ok {
+		log.Fatal("Missing PATTERN environment variable")
+	}
+	// Log the regex pattern being used.
+	log.Printf("Using PATTERN: %q\n", pattern)
+	// Compile the regex pattern for later use.
+	re = regexp.MustCompile(pattern)
+
+	// Read the MATCH_VALUE environment variable to determine whether to check the record's value.
+	mk, ok := os.LookupEnv("MATCH_VALUE")
+	checkValue = ok && isTrueVar(mk)
+	log.Printf("MATCH_VALUE set to: %t\n", checkValue)
+
+	log.Println("Initialization complete, waiting for records...")
+
+	// Listen for records to be written, calling doRegexFilter() for each record.
+	transform.OnRecordWritten(doRegexFilter)
+}
+
+// The doRegexFilter() function executes each time a new record is written.
+// It checks whether the record's key or value (based on MATCH_VALUE) matches the compiled regex.
+// If it matches, the record is forwarded, if not, it's dropped.
+func doRegexFilter(e transform.WriteEvent, w transform.RecordWriter) error {
+	// This stores the data to be checked (either the key or value).
+	var dataToCheck []byte
+
+	// Depending on the MATCH_VALUE environment variable, decide whether to check the record's key or value.
+	if checkValue {
+		// Use the value of the record if MATCH_VALUE is true.
+		dataToCheck = e.Record().Value
+		log.Printf("Checking record value: %s\n", string(dataToCheck))
+	} else {
+		// Use the key of the record if MATCH_VALUE is false.
+		dataToCheck = e.Record().Key
+		log.Printf("Checking record key: %s\n", string(dataToCheck))
+	}
+
+	// If there is no key or value to check, log and skip the record.
+	if dataToCheck == nil {
+		log.Println("Record has no key/value to check, skipping.")
+		return nil
+	}
+
+	// Check if the data matches the regex pattern.
+	pass := re.Match(dataToCheck)
+	if pass {
+		// If the record matches the pattern, log and write the record to the output topic.
+		log.Printf("Record matched pattern, passing through. Key: %s, Value: %s\n", string(e.Record().Key), string(e.Record().Value))
+		return w.Write(e.Record())
+	} else {
+		// If the record does not match the pattern, log and drop the record.
+		log.Printf("Record did not match pattern, dropping. Key: %s, Value: %s\n", string(e.Record().Key), string(e.Record().Value))
+		// Do not write the record if it doesn't match the pattern.
+		return nil
+	}
+}

package/docker-compose/transform/transform.yaml

@@ -0,0 +1,33 @@
+# Transform metadata used by the rpk transform build command.
+# This metadata file tells rpk:
+# 1) The transform’s display name, which also becomes the base for the .wasm file name.
+# 2) A brief description of what it does.
+# 3) Defaults for environment variables.
+# 4) Input and output topics (if you want to define them here rather than in the deploy command).
+
+# Human-readable name of the transform. rpk transform build uses this for the generated .wasm file.
+name: regex
+
+description: |
+  Filters the input topic to records that only match a regular expression.
+
+  Regular expressions are implemented using Go's regexp library, which uses the syntax of RE2.
+  See the RE2 wiki for allowed syntax: https://github.com/google/re2/wiki/Syntax
+
+  Environment variables:
+  - PATTERN: The regular expression that will match against records (required).
+  - MATCH_VALUE: By default, the regex matches keys, but if set to "true", the regex matches values.
+
+# By default, no input topic is set here. (You can set it in your deploy command if preferred.)
+input-topic: ""
+
+# By default, no output topic is set here. (You can set it in your deploy command if preferred.)
+output-topic: ""
+
+# Indicates the specific TinyGo environment used to compile your transform.
+language: tinygo-no-goroutines
+
+env:
+  # The PATTERN variable must be provided at deploy time.
+  # Example: --var=PATTERN=".*@example.com"
+  PATTERN: '<required>'

package/extension-utils/compute-out.js

@@ -0,0 +1,38 @@
+'use strict'
+
+const { posix: path } = require('node:path')
+
+function computeOut (src) {
+  const { component, version, module: module_, family, relative } = src
+  const outRelative = family === 'page' ? relative.replace(/\.adoc$/, '.html') : relative
+  const { dir: dirname, base: basename, ext: extname, name: stem } = path.parse(outRelative)
+  const componentVersion = this.getComponentVersion(component, version)
+  const versionSegment = componentVersion
+  const outDirSegments = []
+  const moduleRootPathSegments = []
+  if (component !== 'ROOT') outDirSegments.push(component)
+  if (versionSegment) outDirSegments.push(versionSegment)
+  if (module_ !== 'ROOT') outDirSegments.push(module_)
+  const outModuleDirSegments = outDirSegments.slice()
+  if (family !== 'page') {
+    outDirSegments.push(`_${family}s`)
+    moduleRootPathSegments.push('..')
+  }
+  if (dirname) {
+    outDirSegments.push(dirname)
+    for (const _ of dirname.split('/')) moduleRootPathSegments.push('..')
+  }
+  const rootPathSegments = moduleRootPathSegments.slice()
+  for (const _ of outModuleDirSegments) rootPathSegments.push('..')
+  const outDirname = outDirSegments.join('/')
+  const result = {
+    dirname: outDirname,
+    basename,
+    path: outDirname + '/' + basename,
+    moduleRootPath: moduleRootPathSegments.length ? moduleRootPathSegments.join('/') : '.',
+    rootPath: rootPathSegments.length ? rootPathSegments.join('/') : '.',
+  }
+  return result
+}
+
+module.exports = computeOut

package/extension-utils/create-asciidoc-file.js

@@ -0,0 +1,15 @@
+'use strict'
+
+const computeOut = require('./compute-out')
+const { posix: path } = require('node:path')
+
+function createAsciiDocFile (contentCatalog, file) {
+  file.mediaType = 'text/asciidoc'
+  const src = file.src
+  const out = computeOut.call(contentCatalog, src)
+  const pub = { url: '/' + out.path, moduleRootPath: out.moduleRootPath, rootPath: out.rootPath }
+  contentCatalog.removeFile((file = contentCatalog.addFile(Object.assign(file, { path: out.path, out: null, pub: pub }))))
+  return file
+}
+
+module.exports = createAsciiDocFile