openclaw-multi-auto 1.0.9 → 1.1.1

package/scripts/docs-i18n/translator.go

@@ -0,0 +1,247 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	pi "github.com/joshp123/pi-golang"
+)
+
+const (
+	translateMaxAttempts = 3
+	translateBaseDelay   = 15 * time.Second
+)
+
+var errEmptyTranslation = errors.New("empty translation")
+
+type PiTranslator struct {
+	client *pi.OneShotClient
+}
+
+func NewPiTranslator(srcLang, tgtLang string, glossary []GlossaryEntry, thinking string) (*PiTranslator, error) {
+	options := pi.DefaultOneShotOptions()
+	options.AppName = "openclaw-docs-i18n"
+	options.WorkDir = "/tmp"
+	options.Mode = pi.ModeDragons
+	options.Dragons = pi.DragonsOptions{
+		Provider: "anthropic",
+		Model:    modelVersion,
+		Thinking: normalizeThinking(thinking),
+	}
+	options.SystemPrompt = translationPrompt(srcLang, tgtLang, glossary)
+	client, err := pi.StartOneShot(options)
+	if err != nil {
+		return nil, err
+	}
+	return &PiTranslator{client: client}, nil
+}
+
+func (t *PiTranslator) Translate(ctx context.Context, text, srcLang, tgtLang string) (string, error) {
+	return t.translate(ctx, text, t.translateMasked)
+}
+
+func (t *PiTranslator) TranslateRaw(ctx context.Context, text, srcLang, tgtLang string) (string, error) {
+	return t.translate(ctx, text, t.translateRaw)
+}
+
+func (t *PiTranslator) translate(ctx context.Context, text string, run func(context.Context, string) (string, error)) (string, error) {
+	if t.client == nil {
+		return "", errors.New("pi client unavailable")
+	}
+	prefix, core, suffix := splitWhitespace(text)
+	if core == "" {
+		return text, nil
+	}
+	translated, err := t.translateWithRetry(ctx, func(ctx context.Context) (string, error) {
+		return run(ctx, core)
+	})
+	if err != nil {
+		return "", err
+	}
+	return prefix + translated + suffix, nil
+}
+
+func (t *PiTranslator) translateWithRetry(ctx context.Context, run func(context.Context) (string, error)) (string, error) {
+	var lastErr error
+	for attempt := 0; attempt < translateMaxAttempts; attempt++ {
+		translated, err := run(ctx)
+		if err == nil {
+			return translated, nil
+		}
+		if !isRetryableTranslateError(err) {
+			return "", err
+		}
+		lastErr = err
+		if attempt+1 < translateMaxAttempts {
+			delay := translateBaseDelay * time.Duration(attempt+1)
+			if err := sleepWithContext(ctx, delay); err != nil {
+				return "", err
+			}
+		}
+	}
+	return "", lastErr
+}
+
+func (t *PiTranslator) translateMasked(ctx context.Context, core string) (string, error) {
+	state := NewPlaceholderState(core)
+	placeholders := make([]string, 0, 8)
+	mapping := map[string]string{}
+	masked := maskMarkdown(core, state.Next, &placeholders, mapping)
+	resText, err := runPrompt(ctx, t.client, masked)
+	if err != nil {
+		return "", err
+	}
+	translated := strings.TrimSpace(resText)
+	if translated == "" {
+		return "", errEmptyTranslation
+	}
+	if err := validatePlaceholders(translated, placeholders); err != nil {
+		return "", err
+	}
+	return unmaskMarkdown(translated, placeholders, mapping), nil
+}
+
+func (t *PiTranslator) translateRaw(ctx context.Context, core string) (string, error) {
+	resText, err := runPrompt(ctx, t.client, core)
+	if err != nil {
+		return "", err
+	}
+	translated := strings.TrimSpace(resText)
+	if translated == "" {
+		return "", errEmptyTranslation
+	}
+	return translated, nil
+}
+
+func isRetryableTranslateError(err error) bool {
+	if err == nil {
+		return false
+	}
+	if errors.Is(err, errEmptyTranslation) {
+		return true
+	}
+	message := strings.ToLower(err.Error())
+	return strings.Contains(message, "placeholder missing") || strings.Contains(message, "rate limit") || strings.Contains(message, "429")
+}
+
+func sleepWithContext(ctx context.Context, delay time.Duration) error {
+	timer := time.NewTimer(delay)
+	defer timer.Stop()
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case <-timer.C:
+		return nil
+	}
+}
+
+func (t *PiTranslator) Close() {
+	if t.client != nil {
+		_ = t.client.Close()
+	}
+}
+
+type agentEndPayload struct {
+	Messages []agentMessage `json:"messages"`
+}
+
+type agentMessage struct {
+	Role         string          `json:"role"`
+	Content      json.RawMessage `json:"content"`
+	StopReason   string          `json:"stopReason,omitempty"`
+	ErrorMessage string          `json:"errorMessage,omitempty"`
+}
+
+type contentBlock struct {
+	Type string `json:"type"`
+	Text string `json:"text,omitempty"`
+}
+
+func runPrompt(ctx context.Context, client *pi.OneShotClient, message string) (string, error) {
+	events, cancel := client.Subscribe(256)
+	defer cancel()
+
+	if err := client.Prompt(ctx, message); err != nil {
+		return "", err
+	}
+
+	for {
+		select {
+		case <-ctx.Done():
+			return "", ctx.Err()
+		case event, ok := <-events:
+			if !ok {
+				return "", errors.New("event stream closed")
+			}
+			if event.Type == "agent_end" {
+				return extractTranslationResult(event.Raw)
+			}
+		}
+	}
+}
+
+func extractTranslationResult(raw json.RawMessage) (string, error) {
+	var payload agentEndPayload
+	if err := json.Unmarshal(raw, &payload); err != nil {
+		return "", err
+	}
+	for index := len(payload.Messages) - 1; index >= 0; index-- {
+		message := payload.Messages[index]
+		if message.Role != "assistant" {
+			continue
+		}
+		if message.ErrorMessage != "" || strings.EqualFold(message.StopReason, "error") {
+			msg := strings.TrimSpace(message.ErrorMessage)
+			if msg == "" {
+				msg = "unknown error"
+			}
+			return "", fmt.Errorf("pi error: %s", msg)
+		}
+		text, err := extractContentText(message.Content)
+		if err != nil {
+			return "", err
+		}
+		return text, nil
+	}
+	return "", errors.New("assistant message not found")
+}
+
+func extractContentText(content json.RawMessage) (string, error) {
+	trimmed := strings.TrimSpace(string(content))
+	if trimmed == "" {
+		return "", nil
+	}
+	if strings.HasPrefix(trimmed, "\"") {
+		var text string
+		if err := json.Unmarshal(content, &text); err != nil {
+			return "", err
+		}
+		return text, nil
+	}
+
+	var blocks []contentBlock
+	if err := json.Unmarshal(content, &blocks); err != nil {
+		return "", err
+	}
+
+	var parts []string
+	for _, block := range blocks {
+		if block.Type == "text" && block.Text != "" {
+			parts = append(parts, block.Text)
+		}
+	}
+	return strings.Join(parts, ""), nil
+}
+
+func normalizeThinking(value string) string {
+	switch strings.ToLower(strings.TrimSpace(value)) {
+	case "low", "high":
+		return strings.ToLower(strings.TrimSpace(value))
+	default:
+		return "high"
+	}
+}

package/scripts/docs-i18n/util.go

@@ -0,0 +1,81 @@
+package main
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+)
+
+const (
+	workflowVersion = 15
+	providerName    = "pi"
+	modelVersion    = "claude-opus-4-6"
+)
+
+func cacheNamespace() string {
+	return fmt.Sprintf("wf=%d|provider=%s|model=%s", workflowVersion, providerName, modelVersion)
+}
+
+func cacheKey(namespace, srcLang, tgtLang, segmentID, textHash string) string {
+	raw := fmt.Sprintf("%s|%s|%s|%s|%s", namespace, srcLang, tgtLang, segmentID, textHash)
+	hash := sha256.Sum256([]byte(raw))
+	return hex.EncodeToString(hash[:])
+}
+
+func hashText(text string) string {
+	normalized := normalizeText(text)
+	hash := sha256.Sum256([]byte(normalized))
+	return hex.EncodeToString(hash[:])
+}
+
+func hashBytes(data []byte) string {
+	hash := sha256.Sum256(data)
+	return hex.EncodeToString(hash[:])
+}
+
+func normalizeText(text string) string {
+	return strings.Join(strings.Fields(strings.TrimSpace(text)), " ")
+}
+
+func segmentID(relPath, textHash string) string {
+	shortHash := textHash
+	if len(shortHash) > 16 {
+		shortHash = shortHash[:16]
+	}
+	return fmt.Sprintf("%s:%s", relPath, shortHash)
+}
+
+func splitWhitespace(text string) (string, string, string) {
+	if text == "" {
+		return "", "", ""
+	}
+	start := 0
+	for start < len(text) && isWhitespace(text[start]) {
+		start++
+	}
+	end := len(text)
+	for end > start && isWhitespace(text[end-1]) {
+		end--
+	}
+	return text[:start], text[start:end], text[end:]
+}
+
+func isWhitespace(b byte) bool {
+	switch b {
+	case ' ', '\t', '\n', '\r':
+		return true
+	default:
+		return false
+	}
+}
+
+func fatal(err error) {
+	if err == nil {
+		return
+	}
+	_, _ = io.WriteString(os.Stderr, err.Error()+"\n")
+	os.Exit(1)
+}

package/scripts/docs-link-audit.mjs

@@ -0,0 +1,233 @@
+#!/usr/bin/env node
+
+import fs from "node:fs";
+import path from "node:path";
+
+const ROOT = process.cwd();
+const DOCS_DIR = path.join(ROOT, "docs");
+const DOCS_JSON_PATH = path.join(DOCS_DIR, "docs.json");
+
+if (!fs.existsSync(DOCS_DIR) || !fs.statSync(DOCS_DIR).isDirectory()) {
+  console.error("docs:check-links: missing docs directory; run from repo root.");
+  process.exit(1);
+}
+
+if (!fs.existsSync(DOCS_JSON_PATH)) {
+  console.error("docs:check-links: missing docs/docs.json.");
+  process.exit(1);
+}
+
+/** @param {string} dir */
+function walk(dir) {
+  /** @type {string[]} */
+  const out = [];
+  const entries = fs.readdirSync(dir, { withFileTypes: true });
+  for (const entry of entries) {
+    if (entry.name.startsWith(".")) {
+      continue;
+    }
+    const full = path.join(dir, entry.name);
+    if (entry.isDirectory()) {
+      out.push(...walk(full));
+    } else if (entry.isFile()) {
+      out.push(full);
+    }
+  }
+  return out;
+}
+
+/** @param {string} p */
+function normalizeSlashes(p) {
+  return p.replace(/\\/g, "/");
+}
+
+/** @param {string} p */
+function normalizeRoute(p) {
+  const stripped = p.replace(/^\/+|\/+$/g, "");
+  return stripped ? `/${stripped}` : "/";
+}
+
+/** @param {string} text */
+function stripInlineCode(text) {
+  return text.replace(/`[^`]+`/g, "");
+}
+
+const docsConfig = JSON.parse(fs.readFileSync(DOCS_JSON_PATH, "utf8"));
+const redirects = new Map();
+for (const item of docsConfig.redirects || []) {
+  const source = normalizeRoute(String(item.source || ""));
+  const destination = normalizeRoute(String(item.destination || ""));
+  redirects.set(source, destination);
+}
+
+const allFiles = walk(DOCS_DIR);
+const relAllFiles = new Set(allFiles.map((abs) => normalizeSlashes(path.relative(DOCS_DIR, abs))));
+
+const markdownFiles = allFiles.filter((abs) => /\.(md|mdx)$/i.test(abs));
+const routes = new Set();
+
+for (const abs of markdownFiles) {
+  const rel = normalizeSlashes(path.relative(DOCS_DIR, abs));
+  const text = fs.readFileSync(abs, "utf8");
+  const slug = rel.replace(/\.(md|mdx)$/i, "");
+  const route = normalizeRoute(slug);
+  routes.add(route);
+  if (slug.endsWith("/index")) {
+    routes.add(normalizeRoute(slug.slice(0, -"/index".length)));
+  }
+
+  if (!text.startsWith("---")) {
+    continue;
+  }
+
+  const end = text.indexOf("\n---", 3);
+  if (end === -1) {
+    continue;
+  }
+  const frontMatter = text.slice(3, end);
+  const match = frontMatter.match(/^permalink:\s*(.+)\s*$/m);
+  if (!match) {
+    continue;
+  }
+  const permalink = String(match[1])
+    .trim()
+    .replace(/^['"]|['"]$/g, "");
+  routes.add(normalizeRoute(permalink));
+}
+
+/** @param {string} route */
+function resolveRoute(route) {
+  let current = normalizeRoute(route);
+  if (current === "/") {
+    return { ok: true, terminal: "/" };
+  }
+
+  const seen = new Set([current]);
+  while (redirects.has(current)) {
+    current = redirects.get(current);
+    if (seen.has(current)) {
+      return { ok: false, terminal: current, loop: true };
+    }
+    seen.add(current);
+  }
+  return { ok: routes.has(current), terminal: current };
+}
+
+const markdownLinkRegex = /!?\[[^\]]*\]\(([^)]+)\)/g;
+
+/** @type {{file: string; line: number; link: string; reason: string}[]} */
+const broken = [];
+let checked = 0;
+
+for (const abs of markdownFiles) {
+  const rel = normalizeSlashes(path.relative(DOCS_DIR, abs));
+  const baseDir = normalizeSlashes(path.dirname(rel));
+  const rawText = fs.readFileSync(abs, "utf8");
+  const lines = rawText.split("\n");
+
+  // Track if we're inside a code fence
+  let inCodeFence = false;
+
+  for (let lineNum = 0; lineNum < lines.length; lineNum++) {
+    let line = lines[lineNum];
+
+    // Toggle code fence state
+    if (line.trim().startsWith("```")) {
+      inCodeFence = !inCodeFence;
+      continue;
+    }
+    if (inCodeFence) {
+      continue;
+    }
+
+    // Strip inline code to avoid false positives
+    line = stripInlineCode(line);
+
+    for (const match of line.matchAll(markdownLinkRegex)) {
+      const raw = match[1]?.trim();
+      if (!raw) {
+        continue;
+      }
+      // Skip external links, mailto, tel, data, and same-page anchors
+      if (/^(https?:|mailto:|tel:|data:|#)/i.test(raw)) {
+        continue;
+      }
+
+      const [pathPart] = raw.split("#");
+      const clean = pathPart.split("?")[0];
+      if (!clean) {
+        // Same-page anchor only (already skipped above)
+        continue;
+      }
+      checked++;
+
+      if (clean.startsWith("/")) {
+        const route = normalizeRoute(clean);
+        const resolvedRoute = resolveRoute(route);
+        if (!resolvedRoute.ok) {
+          const staticRel = route.replace(/^\//, "");
+          if (!relAllFiles.has(staticRel)) {
+            broken.push({
+              file: rel,
+              line: lineNum + 1,
+              link: raw,
+              reason: `route/file not found (terminal: ${resolvedRoute.terminal})`,
+            });
+            continue;
+          }
+        }
+        // Skip anchor validation - Mintlify generates anchors from MDX components,
+        // accordions, and config schemas that we can't reliably extract from markdown.
+        continue;
+      }
+
+      // Relative placeholder strings used in code examples (for example "url")
+      // are intentionally skipped.
+      if (!clean.startsWith(".") && !clean.includes("/")) {
+        continue;
+      }
+
+      const normalizedRel = normalizeSlashes(path.normalize(path.join(baseDir, clean)));
+
+      if (/\.[a-zA-Z0-9]+$/.test(normalizedRel)) {
+        if (!relAllFiles.has(normalizedRel)) {
+          broken.push({
+            file: rel,
+            line: lineNum + 1,
+            link: raw,
+            reason: "relative file not found",
+          });
+        }
+        continue;
+      }
+
+      const candidates = [
+        normalizedRel,
+        `${normalizedRel}.md`,
+        `${normalizedRel}.mdx`,
+        `${normalizedRel}/index.md`,
+        `${normalizedRel}/index.mdx`,
+      ];
+
+      if (!candidates.some((candidate) => relAllFiles.has(candidate))) {
+        broken.push({
+          file: rel,
+          line: lineNum + 1,
+          link: raw,
+          reason: "relative doc target not found",
+        });
+      }
+    }
+  }
+}
+
+console.log(`checked_internal_links=${checked}`);
+console.log(`broken_links=${broken.length}`);
+
+for (const item of broken) {
+  console.log(`${item.file}:${item.line} :: ${item.link} :: ${item.reason}`);
+}
+
+if (broken.length > 0) {
+  process.exit(1);
+}