openclaw-multi-auto 1.0.9 → 1.1.1

package/scripts/docs-i18n/main.go

@@ -0,0 +1,273 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"sync"
+	"time"
+)
+
+type docJob struct {
+	index int
+	path  string
+	rel   string
+}
+
+type docResult struct {
+	index    int
+	rel      string
+	duration time.Duration
+	skipped  bool
+	err      error
+}
+
+func main() {
+	var (
+		targetLang = flag.String("lang", "zh-CN", "target language (e.g., zh-CN)")
+		sourceLang = flag.String("src", "en", "source language")
+		docsRoot   = flag.String("docs", "docs", "docs root")
+		tmPath     = flag.String("tm", "", "translation memory path")
+		mode       = flag.String("mode", "segment", "translation mode (segment|doc)")
+		thinking   = flag.String("thinking", "high", "thinking level (low|high)")
+		overwrite  = flag.Bool("overwrite", false, "overwrite existing translations")
+		maxFiles   = flag.Int("max", 0, "max files to process (0 = all)")
+		parallel   = flag.Int("parallel", 1, "parallel workers for doc mode")
+	)
+	flag.Parse()
+	files := flag.Args()
+	if len(files) == 0 {
+		fatal(fmt.Errorf("no doc files provided"))
+	}
+
+	resolvedDocsRoot, err := filepath.Abs(*docsRoot)
+	if err != nil {
+		fatal(err)
+	}
+
+	if *tmPath == "" {
+		*tmPath = filepath.Join(resolvedDocsRoot, ".i18n", fmt.Sprintf("%s.tm.jsonl", *targetLang))
+	}
+
+	glossaryPath := filepath.Join(resolvedDocsRoot, ".i18n", fmt.Sprintf("glossary.%s.json", *targetLang))
+	glossary, err := LoadGlossary(glossaryPath)
+	if err != nil {
+		fatal(err)
+	}
+
+	translator, err := NewPiTranslator(*sourceLang, *targetLang, glossary, *thinking)
+	if err != nil {
+		fatal(err)
+	}
+	defer translator.Close()
+
+	tm, err := LoadTranslationMemory(*tmPath)
+	if err != nil {
+		fatal(err)
+	}
+
+	ordered, err := orderFiles(resolvedDocsRoot, files)
+	if err != nil {
+		fatal(err)
+	}
+	totalFiles := len(ordered)
+	preSkipped := 0
+	if *mode == "doc" && !*overwrite {
+		filtered, skipped, err := filterDocQueue(resolvedDocsRoot, *targetLang, ordered)
+		if err != nil {
+			fatal(err)
+		}
+		ordered = filtered
+		preSkipped = skipped
+	}
+	if *maxFiles > 0 && *maxFiles < len(ordered) {
+		ordered = ordered[:*maxFiles]
+	}
+
+	log.SetFlags(log.LstdFlags)
+	start := time.Now()
+	processed := 0
+	skipped := 0
+
+	if *parallel < 1 {
+		*parallel = 1
+	}
+
+	log.Printf("docs-i18n: mode=%s total=%d pending=%d pre_skipped=%d overwrite=%t thinking=%s parallel=%d", *mode, totalFiles, len(ordered), preSkipped, *overwrite, *thinking, *parallel)
+	switch *mode {
+	case "doc":
+		if *parallel > 1 {
+			proc, skip, err := runDocParallel(context.Background(), ordered, resolvedDocsRoot, *sourceLang, *targetLang, *overwrite, *parallel, glossary, *thinking)
+			if err != nil {
+				fatal(err)
+			}
+			processed += proc
+			skipped += skip
+		} else {
+			proc, skip, err := runDocSequential(context.Background(), ordered, translator, resolvedDocsRoot, *sourceLang, *targetLang, *overwrite)
+			if err != nil {
+				fatal(err)
+			}
+			processed += proc
+			skipped += skip
+		}
+	case "segment":
+		if *parallel > 1 {
+			fatal(fmt.Errorf("parallel processing is only supported in doc mode"))
+		}
+		proc, err := runSegmentSequential(context.Background(), ordered, translator, tm, resolvedDocsRoot, *sourceLang, *targetLang)
+		if err != nil {
+			fatal(err)
+		}
+		processed += proc
+	default:
+		fatal(fmt.Errorf("unknown mode: %s", *mode))
+	}
+
+	if err := tm.Save(); err != nil {
+		fatal(err)
+	}
+	elapsed := time.Since(start).Round(time.Millisecond)
+	log.Printf("docs-i18n: completed processed=%d skipped=%d elapsed=%s", processed, skipped, elapsed)
+}
+
+func runDocSequential(ctx context.Context, ordered []string, translator *PiTranslator, docsRoot, srcLang, tgtLang string, overwrite bool) (int, int, error) {
+	processed := 0
+	skipped := 0
+	for index, file := range ordered {
+		relPath := resolveRelPath(docsRoot, file)
+		log.Printf("docs-i18n: [%d/%d] start %s", index+1, len(ordered), relPath)
+		start := time.Now()
+		skip, err := processFileDoc(ctx, translator, docsRoot, file, srcLang, tgtLang, overwrite)
+		if err != nil {
+			return processed, skipped, err
+		}
+		if skip {
+			skipped++
+			log.Printf("docs-i18n: [%d/%d] skipped %s (%s)", index+1, len(ordered), relPath, time.Since(start).Round(time.Millisecond))
+		} else {
+			processed++
+			log.Printf("docs-i18n: [%d/%d] done %s (%s)", index+1, len(ordered), relPath, time.Since(start).Round(time.Millisecond))
+		}
+	}
+	return processed, skipped, nil
+}
+
+func runDocParallel(ctx context.Context, ordered []string, docsRoot, srcLang, tgtLang string, overwrite bool, parallel int, glossary []GlossaryEntry, thinking string) (int, int, error) {
+	jobs := make(chan docJob)
+	results := make(chan docResult, len(ordered))
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	var wg sync.WaitGroup
+	for worker := 0; worker < parallel; worker++ {
+		wg.Add(1)
+		go func(workerID int) {
+			defer wg.Done()
+			translator, err := NewPiTranslator(srcLang, tgtLang, glossary, thinking)
+			if err != nil {
+				results <- docResult{err: err}
+				return
+			}
+			defer translator.Close()
+			for job := range jobs {
+				if ctx.Err() != nil {
+					return
+				}
+				log.Printf("docs-i18n: [w%d %d/%d] start %s", workerID, job.index, len(ordered), job.rel)
+				start := time.Now()
+				skip, err := processFileDoc(ctx, translator, docsRoot, job.path, srcLang, tgtLang, overwrite)
+				results <- docResult{
+					index:    job.index,
+					rel:      job.rel,
+					duration: time.Since(start),
+					skipped:  skip,
+					err:      err,
+				}
+				if err != nil {
+					cancel()
+					return
+				}
+			}
+		}(worker + 1)
+	}
+
+	go func() {
+		for index, file := range ordered {
+			jobs <- docJob{index: index + 1, path: file, rel: resolveRelPath(docsRoot, file)}
+		}
+		close(jobs)
+	}()
+
+	processed := 0
+	skipped := 0
+	for i := 0; i < len(ordered); i++ {
+		result := <-results
+		if result.err != nil {
+			wg.Wait()
+			return processed, skipped, result.err
+		}
+		if result.skipped {
+			skipped++
+			log.Printf("docs-i18n: [w* %d/%d] skipped %s (%s)", result.index, len(ordered), result.rel, result.duration.Round(time.Millisecond))
+		} else {
+			processed++
+			log.Printf("docs-i18n: [w* %d/%d] done %s (%s)", result.index, len(ordered), result.rel, result.duration.Round(time.Millisecond))
+		}
+	}
+	wg.Wait()
+	return processed, skipped, nil
+}
+
+func runSegmentSequential(ctx context.Context, ordered []string, translator *PiTranslator, tm *TranslationMemory, docsRoot, srcLang, tgtLang string) (int, error) {
+	processed := 0
+	for index, file := range ordered {
+		relPath := resolveRelPath(docsRoot, file)
+		log.Printf("docs-i18n: [%d/%d] start %s", index+1, len(ordered), relPath)
+		start := time.Now()
+		if _, err := processFile(ctx, translator, tm, docsRoot, file, srcLang, tgtLang); err != nil {
+			return processed, err
+		}
+		processed++
+		log.Printf("docs-i18n: [%d/%d] done %s (%s)", index+1, len(ordered), relPath, time.Since(start).Round(time.Millisecond))
+	}
+	return processed, nil
+}
+
+func resolveRelPath(docsRoot, file string) string {
+	relPath := file
+	if _, rel, err := resolveDocsPath(docsRoot, file); err == nil {
+		relPath = rel
+	}
+	return relPath
+}
+
+func filterDocQueue(docsRoot, targetLang string, ordered []string) ([]string, int, error) {
+	pending := make([]string, 0, len(ordered))
+	skipped := 0
+	for _, file := range ordered {
+		absPath, relPath, err := resolveDocsPath(docsRoot, file)
+		if err != nil {
+			return nil, skipped, err
+		}
+		content, err := os.ReadFile(absPath)
+		if err != nil {
+			return nil, skipped, err
+		}
+		sourceHash := hashBytes(content)
+		outputPath := filepath.Join(docsRoot, targetLang, relPath)
+		skip, err := shouldSkipDoc(outputPath, sourceHash)
+		if err != nil {
+			return nil, skipped, err
+		}
+		if skip {
+			skipped++
+			continue
+		}
+		pending = append(pending, file)
+	}
+	return pending, skipped, nil
+}

package/scripts/docs-i18n/markdown_segments.go

@@ -0,0 +1,131 @@
+package main
+
+import (
+	"sort"
+	"strings"
+
+	"github.com/yuin/goldmark"
+	"github.com/yuin/goldmark/ast"
+	"github.com/yuin/goldmark/extension"
+	"github.com/yuin/goldmark/text"
+)
+
+func extractSegments(body, relPath string) ([]Segment, error) {
+	source := []byte(body)
+	r := text.NewReader(source)
+	md := goldmark.New(
+		goldmark.WithExtensions(extension.GFM),
+	)
+	doc := md.Parser().Parse(r)
+
+	segments := make([]Segment, 0, 128)
+	skipDepth := 0
+	var lastBlock ast.Node
+
+	err := ast.Walk(doc, func(n ast.Node, entering bool) (ast.WalkStatus, error) {
+		switch n.(type) {
+		case *ast.CodeBlock, *ast.FencedCodeBlock, *ast.CodeSpan, *ast.HTMLBlock, *ast.RawHTML:
+			if entering {
+				skipDepth++
+			} else {
+				skipDepth--
+			}
+			return ast.WalkContinue, nil
+		}
+
+		if !entering || skipDepth > 0 {
+			return ast.WalkContinue, nil
+		}
+
+		textNode, ok := n.(*ast.Text)
+		if !ok {
+			return ast.WalkContinue, nil
+		}
+		block := blockParent(textNode)
+		if block == nil {
+			return ast.WalkContinue, nil
+		}
+		textValue := string(textNode.Segment.Value(source))
+		if strings.TrimSpace(textValue) == "" {
+			return ast.WalkContinue, nil
+		}
+
+		start := textNode.Segment.Start
+		stop := textNode.Segment.Stop
+		if len(segments) > 0 && lastBlock == block {
+			last := &segments[len(segments)-1]
+			gap := string(source[last.Stop:start])
+			if strings.TrimSpace(gap) == "" {
+				last.Stop = stop
+				return ast.WalkContinue, nil
+			}
+		}
+
+		segments = append(segments, Segment{Start: start, Stop: stop})
+		lastBlock = block
+		return ast.WalkContinue, nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	filtered := make([]Segment, 0, len(segments))
+	for _, seg := range segments {
+		textValue := string(source[seg.Start:seg.Stop])
+		trimmed := strings.TrimSpace(textValue)
+		if trimmed == "" {
+			continue
+		}
+		textHash := hashText(textValue)
+		segmentID := segmentID(relPath, textHash)
+		filtered = append(filtered, Segment{
+			Start:     seg.Start,
+			Stop:      seg.Stop,
+			Text:      textValue,
+			TextHash:  textHash,
+			SegmentID: segmentID,
+		})
+	}
+
+	sort.Slice(filtered, func(i, j int) bool {
+		return filtered[i].Start < filtered[j].Start
+	})
+
+	return filtered, nil
+}
+
+func blockParent(n ast.Node) ast.Node {
+	for node := n.Parent(); node != nil; node = node.Parent() {
+		if isTranslatableBlock(node) {
+			return node
+		}
+	}
+	return nil
+}
+
+func isTranslatableBlock(n ast.Node) bool {
+	switch n.(type) {
+	case *ast.Paragraph, *ast.Heading, *ast.ListItem:
+		return true
+	default:
+		return false
+	}
+}
+
+func applyTranslations(body string, segments []Segment) string {
+	if len(segments) == 0 {
+		return body
+	}
+	var out strings.Builder
+	last := 0
+	for _, seg := range segments {
+		if seg.Start < last {
+			continue
+		}
+		out.WriteString(body[last:seg.Start])
+		out.WriteString(seg.Translated)
+		last = seg.Stop
+	}
+	out.WriteString(body[last:])
+	return out.String()
+}

package/scripts/docs-i18n/masking.go

@@ -0,0 +1,89 @@
+package main
+
+import (
+	"fmt"
+	"regexp"
+	"strings"
+)
+
+var (
+	inlineCodeRe  = regexp.MustCompile("`[^`]+`")
+	angleLinkRe   = regexp.MustCompile(`<https?://[^>]+>`)
+	linkURLRe     = regexp.MustCompile(`\[[^\]]*\]\(([^)]+)\)`)
+	placeholderRe = regexp.MustCompile(`__OC_I18N_\d+__`)
+)
+
+func maskMarkdown(text string, nextPlaceholder func() string, placeholders *[]string, mapping map[string]string) string {
+	masked := maskMatches(text, inlineCodeRe, nextPlaceholder, placeholders, mapping)
+	masked = maskMatches(masked, angleLinkRe, nextPlaceholder, placeholders, mapping)
+	masked = maskLinkURLs(masked, nextPlaceholder, placeholders, mapping)
+	return masked
+}
+
+func maskMatches(text string, re *regexp.Regexp, nextPlaceholder func() string, placeholders *[]string, mapping map[string]string) string {
+	matches := re.FindAllStringIndex(text, -1)
+	if len(matches) == 0 {
+		return text
+	}
+	var out strings.Builder
+	pos := 0
+	for _, span := range matches {
+		start, end := span[0], span[1]
+		if start < pos {
+			continue
+		}
+		out.WriteString(text[pos:start])
+		placeholder := nextPlaceholder()
+		mapping[placeholder] = text[start:end]
+		*placeholders = append(*placeholders, placeholder)
+		out.WriteString(placeholder)
+		pos = end
+	}
+	out.WriteString(text[pos:])
+	return out.String()
+}
+
+func maskLinkURLs(text string, nextPlaceholder func() string, placeholders *[]string, mapping map[string]string) string {
+	matches := linkURLRe.FindAllStringSubmatchIndex(text, -1)
+	if len(matches) == 0 {
+		return text
+	}
+	var out strings.Builder
+	pos := 0
+	for _, span := range matches {
+		fullStart := span[0]
+		urlStart, urlEnd := span[2], span[3]
+		if urlStart < 0 || urlEnd < 0 {
+			continue
+		}
+		if fullStart < pos {
+			continue
+		}
+		out.WriteString(text[pos:urlStart])
+		placeholder := nextPlaceholder()
+		mapping[placeholder] = text[urlStart:urlEnd]
+		*placeholders = append(*placeholders, placeholder)
+		out.WriteString(placeholder)
+		pos = urlEnd
+	}
+	out.WriteString(text[pos:])
+	return out.String()
+}
+
+func unmaskMarkdown(text string, placeholders []string, mapping map[string]string) string {
+	out := text
+	for _, placeholder := range placeholders {
+		original := mapping[placeholder]
+		out = strings.ReplaceAll(out, placeholder, original)
+	}
+	return out
+}
+
+func validatePlaceholders(text string, placeholders []string) error {
+	for _, placeholder := range placeholders {
+		if !strings.Contains(text, placeholder) {
+			return fmt.Errorf("placeholder missing: %s", placeholder)
+		}
+	}
+	return nil
+}

package/scripts/docs-i18n/order.go

@@ -0,0 +1,37 @@
+package main
+
+import (
+	"path/filepath"
+	"sort"
+)
+
+type orderedFile struct {
+	path string
+	rel  string
+}
+
+func orderFiles(docsRoot string, files []string) ([]string, error) {
+	entries := make([]orderedFile, 0, len(files))
+	for _, file := range files {
+		abs, err := filepath.Abs(file)
+		if err != nil {
+			return nil, err
+		}
+		rel, err := filepath.Rel(docsRoot, abs)
+		if err != nil {
+			rel = abs
+		}
+		entries = append(entries, orderedFile{path: file, rel: rel})
+	}
+	if len(entries) == 0 {
+		return nil, nil
+	}
+	sort.Slice(entries, func(i, j int) bool {
+		return entries[i].rel < entries[j].rel
+	})
+	ordered := make([]string, 0, len(entries))
+	for _, entry := range entries {
+		ordered = append(ordered, entry.path)
+	}
+	return ordered, nil
+}

package/scripts/docs-i18n/placeholders.go

@@ -0,0 +1,30 @@
+package main
+
+import (
+	"fmt"
+)
+
+type PlaceholderState struct {
+	counter int
+	used    map[string]struct{}
+}
+
+func NewPlaceholderState(text string) *PlaceholderState {
+	used := map[string]struct{}{}
+	for _, hit := range placeholderRe.FindAllString(text, -1) {
+		used[hit] = struct{}{}
+	}
+	return &PlaceholderState{counter: 900000, used: used}
+}
+
+func (s *PlaceholderState) Next() string {
+	for {
+		candidate := fmt.Sprintf("__OC_I18N_%d__", s.counter)
+		s.counter++
+		if _, ok := s.used[candidate]; ok {
+			continue
+		}
+		s.used[candidate] = struct{}{}
+		return candidate
+	}
+}