openclaw-multi-auto 1.0.9 → 1.1.1

package/scripts/docs-i18n/process.go

@@ -0,0 +1,202 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"gopkg.in/yaml.v3"
+)
+
+func processFile(ctx context.Context, translator *PiTranslator, tm *TranslationMemory, docsRoot, filePath, srcLang, tgtLang string) (bool, error) {
+	absPath, relPath, err := resolveDocsPath(docsRoot, filePath)
+	if err != nil {
+		return false, err
+	}
+
+	content, err := os.ReadFile(absPath)
+	if err != nil {
+		return false, err
+	}
+
+	frontMatter, body := splitFrontMatter(string(content))
+	frontData := map[string]any{}
+	if frontMatter != "" {
+		if err := yaml.Unmarshal([]byte(frontMatter), &frontData); err != nil {
+			return false, fmt.Errorf("frontmatter parse failed for %s: %w", relPath, err)
+		}
+	}
+
+	if err := translateFrontMatter(ctx, translator, tm, frontData, relPath, srcLang, tgtLang); err != nil {
+		return false, err
+	}
+
+	body, err = translateHTMLBlocks(ctx, translator, body, srcLang, tgtLang)
+	if err != nil {
+		return false, err
+	}
+
+	segments, err := extractSegments(body, relPath)
+	if err != nil {
+		return false, err
+	}
+
+	namespace := cacheNamespace()
+	for i := range segments {
+		seg := &segments[i]
+		seg.CacheKey = cacheKey(namespace, srcLang, tgtLang, seg.SegmentID, seg.TextHash)
+		if entry, ok := tm.Get(seg.CacheKey); ok {
+			seg.Translated = entry.Translated
+			continue
+		}
+		translated, err := translator.Translate(ctx, seg.Text, srcLang, tgtLang)
+		if err != nil {
+			return false, fmt.Errorf("translate failed (%s): %w", relPath, err)
+		}
+		seg.Translated = translated
+		entry := TMEntry{
+			CacheKey:   seg.CacheKey,
+			SegmentID:  seg.SegmentID,
+			SourcePath: relPath,
+			TextHash:   seg.TextHash,
+			Text:       seg.Text,
+			Translated: translated,
+			Provider:   providerName,
+			Model:      modelVersion,
+			SrcLang:    srcLang,
+			TgtLang:    tgtLang,
+			UpdatedAt:  time.Now().UTC().Format(time.RFC3339),
+		}
+		tm.Put(entry)
+	}
+
+	translatedBody := applyTranslations(body, segments)
+	updatedFront, err := encodeFrontMatter(frontData, relPath, content)
+	if err != nil {
+		return false, err
+	}
+
+	outputPath := filepath.Join(docsRoot, tgtLang, relPath)
+	if err := os.MkdirAll(filepath.Dir(outputPath), 0o755); err != nil {
+		return false, err
+	}
+
+	output := updatedFront + translatedBody
+	return false, os.WriteFile(outputPath, []byte(output), 0o644)
+}
+
+func splitFrontMatter(content string) (string, string) {
+	if !strings.HasPrefix(content, "---") {
+		return "", content
+	}
+	lines := strings.Split(content, "\n")
+	if len(lines) < 2 {
+		return "", content
+	}
+	endIndex := -1
+	for i := 1; i < len(lines); i++ {
+		if strings.TrimSpace(lines[i]) == "---" {
+			endIndex = i
+			break
+		}
+	}
+	if endIndex == -1 {
+		return "", content
+	}
+	front := strings.Join(lines[1:endIndex], "\n")
+	body := strings.Join(lines[endIndex+1:], "\n")
+	if strings.HasPrefix(body, "\n") {
+		body = body[1:]
+	}
+	return front, body
+}
+
+func encodeFrontMatter(frontData map[string]any, relPath string, source []byte) (string, error) {
+	if frontData == nil {
+		frontData = map[string]any{}
+	}
+	frontData["x-i18n"] = map[string]any{
+		"source_path":  relPath,
+		"source_hash":  hashBytes(source),
+		"provider":     providerName,
+		"model":        modelVersion,
+		"workflow":     workflowVersion,
+		"generated_at": time.Now().UTC().Format(time.RFC3339),
+	}
+	encoded, err := yaml.Marshal(frontData)
+	if err != nil {
+		return "", err
+	}
+	return fmt.Sprintf("---\n%s---\n\n", string(encoded)), nil
+}
+
+func translateFrontMatter(ctx context.Context, translator *PiTranslator, tm *TranslationMemory, data map[string]any, relPath, srcLang, tgtLang string) error {
+	if len(data) == 0 {
+		return nil
+	}
+	if summary, ok := data["summary"].(string); ok {
+		translated, err := translateSnippet(ctx, translator, tm, relPath+":frontmatter:summary", summary, srcLang, tgtLang)
+		if err != nil {
+			return err
+		}
+		data["summary"] = translated
+	}
+	if title, ok := data["title"].(string); ok {
+		translated, err := translateSnippet(ctx, translator, tm, relPath+":frontmatter:title", title, srcLang, tgtLang)
+		if err != nil {
+			return err
+		}
+		data["title"] = translated
+	}
+	if readWhen, ok := data["read_when"].([]any); ok {
+		translated := make([]any, 0, len(readWhen))
+		for idx, item := range readWhen {
+			textValue, ok := item.(string)
+			if !ok {
+				translated = append(translated, item)
+				continue
+			}
+			value, err := translateSnippet(ctx, translator, tm, fmt.Sprintf("%s:frontmatter:read_when:%d", relPath, idx), textValue, srcLang, tgtLang)
+			if err != nil {
+				return err
+			}
+			translated = append(translated, value)
+		}
+		data["read_when"] = translated
+	}
+	return nil
+}
+
+func translateSnippet(ctx context.Context, translator *PiTranslator, tm *TranslationMemory, segmentID, textValue, srcLang, tgtLang string) (string, error) {
+	if strings.TrimSpace(textValue) == "" {
+		return textValue, nil
+	}
+	namespace := cacheNamespace()
+	textHash := hashText(textValue)
+	ck := cacheKey(namespace, srcLang, tgtLang, segmentID, textHash)
+	if entry, ok := tm.Get(ck); ok {
+		return entry.Translated, nil
+	}
+	translated, err := translator.Translate(ctx, textValue, srcLang, tgtLang)
+	if err != nil {
+		return "", err
+	}
+	entry := TMEntry{
+		CacheKey:   ck,
+		SegmentID:  segmentID,
+		SourcePath: segmentID,
+		TextHash:   textHash,
+		Text:       textValue,
+		Translated: translated,
+		Provider:   providerName,
+		Model:      modelVersion,
+		SrcLang:    srcLang,
+		TgtLang:    tgtLang,
+		UpdatedAt:  time.Now().UTC().Format(time.RFC3339),
+	}
+	tm.Put(entry)
+	return translated, nil
+}

package/scripts/docs-i18n/prompt.go

@@ -0,0 +1,146 @@
+package main
+
+import (
+	"fmt"
+	"strings"
+)
+
+func prettyLanguageLabel(lang string) string {
+	trimmed := strings.TrimSpace(lang)
+	if trimmed == "" {
+		return lang
+	}
+	switch {
+	case strings.EqualFold(trimmed, "en"):
+		return "English"
+	case strings.EqualFold(trimmed, "zh-CN"):
+		return "Simplified Chinese"
+	case strings.EqualFold(trimmed, "ja-JP"):
+		return "Japanese"
+	default:
+		return trimmed
+	}
+}
+
+func translationPrompt(srcLang, tgtLang string, glossary []GlossaryEntry) string {
+	srcLabel := prettyLanguageLabel(srcLang)
+	tgtLabel := prettyLanguageLabel(tgtLang)
+	glossaryBlock := buildGlossaryPrompt(glossary)
+
+	switch {
+	case strings.EqualFold(tgtLang, "zh-CN"):
+		// Keep this prompt as stable as possible; it has lots of tuning baked into the wording.
+		return strings.TrimSpace(fmt.Sprintf(zhCNPromptTemplate, srcLabel, tgtLabel, glossaryBlock))
+	case strings.EqualFold(tgtLang, "ja-JP"):
+		return strings.TrimSpace(fmt.Sprintf(jaJPPromptTemplate, srcLabel, tgtLabel, glossaryBlock))
+	default:
+		return strings.TrimSpace(fmt.Sprintf(genericPromptTemplate, srcLabel, tgtLabel, glossaryBlock))
+	}
+}
+
+const zhCNPromptTemplate = `You are a translation function, not a chat assistant.
+Translate from %s to %s.
+
+Rules:
+- Output ONLY the translated text. No preamble, no questions, no commentary.
+- Translate all English prose; do not leave English unless it is code, a URL, or a product name.
+- All prose must be Chinese. If any English sentence remains outside code/URLs/product names, it is wrong.
+- If the input contains <frontmatter> and <body> tags, keep them exactly and output exactly one of each.
+- Translate only the contents inside those tags.
+- Preserve YAML structure inside <frontmatter>; translate only values.
+- Preserve all [[[FM_*]]] markers exactly and translate only the text between each START/END pair.
+- Translate headings/labels like "Exit codes" and "Optional scripts".
+- Preserve Markdown syntax exactly (headings, lists, tables, emphasis).
+- Preserve HTML tags and attributes exactly.
+- Do not translate code spans/blocks, config keys, CLI flags, or env vars.
+- Do not alter URLs or anchors.
+- Preserve placeholders exactly: __OC_I18N_####__.
+- Do not remove, reorder, or summarize content.
+- Use fluent, idiomatic technical Chinese; avoid slang or jokes.
+- Use neutral documentation tone; prefer “你/你的”, avoid “您/您的”.
+- Insert a space between Latin characters and CJK text (W3C CLREQ), e.g., “Gateway 网关”, “Skills 配置”.
+- Use Chinese quotation marks “ and ” for Chinese prose; keep ASCII quotes inside code spans/blocks or literal CLI/keys.
+- Keep product names in English: OpenClaw, Pi, WhatsApp, Telegram, Discord, iMessage, Slack, Microsoft Teams, Google Chat, Signal.
+- For the OpenClaw Gateway, use “Gateway 网关”.
+- Keep these terms in English: Skills, local loopback, Tailscale.
+- Never output an empty response; if unsure, return the source text unchanged.
+
+%s
+
+If the input is empty, output empty.
+If the input contains only placeholders, output it unchanged.`
+
+const jaJPPromptTemplate = `You are a translation function, not a chat assistant.
+Translate from %s to %s.
+
+Rules:
+- Output ONLY the translated text. No preamble, no questions, no commentary.
+- Translate all English prose; do not leave English unless it is code, a URL, or a product name.
+- All prose must be Japanese. If any English sentence remains outside code/URLs/product names, it is wrong.
+- If the input contains <frontmatter> and <body> tags, keep them exactly and output exactly one of each.
+- Translate only the contents inside those tags.
+- Preserve YAML structure inside <frontmatter>; translate only values.
+- Preserve all [[[FM_*]]] markers exactly and translate only the text between each START/END pair.
+- Translate headings/labels like "Exit codes" and "Optional scripts".
+- Preserve Markdown syntax exactly (headings, lists, tables, emphasis).
+- Preserve HTML tags and attributes exactly.
+- Do not translate code spans/blocks, config keys, CLI flags, or env vars.
+- Do not alter URLs or anchors.
+- Preserve placeholders exactly: __OC_I18N_####__.
+- Do not remove, reorder, or summarize content.
+- Use fluent, idiomatic technical Japanese; avoid slang or jokes.
+- Use neutral documentation tone; avoid overly formal honorifics (e.g., avoid “〜でございます”).
+- Use Japanese quotation marks 「 and 」 for Japanese prose; keep ASCII quotes inside code spans/blocks or literal CLI/keys.
+- Do not add or remove spacing around Latin text just because it borders Japanese; keep spacing stable unless required by Japanese grammar.
+- Keep product names in English: OpenClaw, Pi, WhatsApp, Telegram, Discord, iMessage, Slack, Microsoft Teams, Google Chat, Signal.
+- Keep these terms in English: Skills, local loopback, Tailscale.
+- Never output an empty response; if unsure, return the source text unchanged.
+
+%s
+
+If the input is empty, output empty.
+If the input contains only placeholders, output it unchanged.`
+
+const genericPromptTemplate = `You are a translation function, not a chat assistant.
+Translate from %s to %s.
+
+Rules:
+- Output ONLY the translated text. No preamble, no questions, no commentary.
+- Translate all English prose; do not leave English unless it is code, a URL, or a product name.
+- If any English sentence remains outside code/URLs/product names, it is likely wrong.
+- If the input contains <frontmatter> and <body> tags, keep them exactly and output exactly one of each.
+- Translate only the contents inside those tags.
+- Preserve YAML structure inside <frontmatter>; translate only values.
+- Preserve all [[[FM_*]]] markers exactly and translate only the text between each START/END pair.
+- Translate headings/labels like "Exit codes" and "Optional scripts".
+- Preserve Markdown syntax exactly (headings, lists, tables, emphasis).
+- Preserve HTML tags and attributes exactly.
+- Do not translate code spans/blocks, config keys, CLI flags, or env vars.
+- Do not alter URLs or anchors.
+- Preserve placeholders exactly: __OC_I18N_####__.
+- Do not remove, reorder, or summarize content.
+- Use fluent, idiomatic technical language in the target language; avoid slang or jokes.
+- Use neutral documentation tone.
+- Keep product names in English: OpenClaw, Pi, WhatsApp, Telegram, Discord, iMessage, Slack, Microsoft Teams, Google Chat, Signal.
+- Keep these terms in English: Skills, local loopback, Tailscale.
+- Never output an empty response; if unsure, return the source text unchanged.
+
+%s
+
+If the input is empty, output empty.
+If the input contains only placeholders, output it unchanged.`
+
+func buildGlossaryPrompt(glossary []GlossaryEntry) string {
+	if len(glossary) == 0 {
+		return ""
+	}
+	var lines []string
+	lines = append(lines, "Preferred translations (use when natural):")
+	for _, entry := range glossary {
+		if entry.Source == "" || entry.Target == "" {
+			continue
+		}
+		lines = append(lines, fmt.Sprintf("- %s -> %s", entry.Source, entry.Target))
+	}
+	return strings.Join(lines, "\n")
+}

package/scripts/docs-i18n/segment.go

@@ -0,0 +1,11 @@
+package main
+
+type Segment struct {
+	Start      int
+	Stop       int
+	Text       string
+	TextHash   string
+	SegmentID  string
+	Translated string
+	CacheKey   string
+}

package/scripts/docs-i18n/tm.go

@@ -0,0 +1,132 @@
+package main
+
+import (
+	"bufio"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+)
+
+type TMEntry struct {
+	CacheKey   string `json:"cache_key"`
+	SegmentID  string `json:"segment_id"`
+	SourcePath string `json:"source_path"`
+	TextHash   string `json:"text_hash"`
+	Text       string `json:"text"`
+	Translated string `json:"translated"`
+	Provider   string `json:"provider"`
+	Model      string `json:"model"`
+	SrcLang    string `json:"src_lang"`
+	TgtLang    string `json:"tgt_lang"`
+	UpdatedAt  string `json:"updated_at"`
+}
+
+type TranslationMemory struct {
+	path    string
+	entries map[string]TMEntry
+}
+
+func LoadTranslationMemory(path string) (*TranslationMemory, error) {
+	tm := &TranslationMemory{path: path, entries: map[string]TMEntry{}}
+	file, err := os.Open(path)
+	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			return tm, nil
+		}
+		return nil, err
+	}
+	defer file.Close()
+
+	reader := bufio.NewReader(file)
+	for {
+		line, err := reader.ReadBytes('\n')
+		if len(line) > 0 {
+			trimmed := strings.TrimSpace(string(line))
+			if trimmed != "" {
+				var entry TMEntry
+				if err := json.Unmarshal([]byte(trimmed), &entry); err != nil {
+					return nil, fmt.Errorf("translation memory decode failed: %w", err)
+				}
+				if entry.CacheKey != "" && strings.TrimSpace(entry.Translated) != "" {
+					tm.entries[entry.CacheKey] = entry
+				}
+			}
+		}
+		if err != nil {
+			if errors.Is(err, io.EOF) {
+				break
+			}
+			return nil, err
+		}
+	}
+	return tm, nil
+}
+
+func (tm *TranslationMemory) Get(cacheKey string) (TMEntry, bool) {
+	entry, ok := tm.entries[cacheKey]
+	if !ok {
+		return TMEntry{}, false
+	}
+	if strings.TrimSpace(entry.Translated) == "" {
+		return TMEntry{}, false
+	}
+	return entry, true
+}
+
+func (tm *TranslationMemory) Put(entry TMEntry) {
+	if entry.CacheKey == "" {
+		return
+	}
+	tm.entries[entry.CacheKey] = entry
+}
+
+func (tm *TranslationMemory) Save() error {
+	if tm.path == "" {
+		return nil
+	}
+	if err := os.MkdirAll(filepath.Dir(tm.path), 0o755); err != nil {
+		return err
+	}
+	tmpPath := tm.path + ".tmp"
+	file, err := os.Create(tmpPath)
+	if err != nil {
+		return err
+	}
+
+	keys := make([]string, 0, len(tm.entries))
+	for key := range tm.entries {
+		keys = append(keys, key)
+	}
+	sort.Strings(keys)
+
+	writer := bufio.NewWriter(file)
+	for _, key := range keys {
+		entry := tm.entries[key]
+		payload, err := json.Marshal(entry)
+		if err != nil {
+			_ = file.Close()
+			return err
+		}
+		if _, err := writer.Write(payload); err != nil {
+			_ = file.Close()
+			return err
+		}
+		if _, err := writer.WriteString("\n"); err != nil {
+			_ = file.Close()
+			return err
+		}
+	}
+	if err := writer.Flush(); err != nil {
+		_ = file.Close()
+		return err
+	}
+	if err := file.Close(); err != nil {
+		return err
+	}
+	return os.Rename(tmpPath, tm.path)
+}