npm - openclaw-multi-auto - Versions diffs - 1.0.9 → 1.1.1 - Mend

openclaw-multi-auto 1.0.9 → 1.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (490) hide show

package/scripts/docs-i18n/doc_mode.go ADDED Viewed

@@ -0,0 +1,272 @@
+package main
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"gopkg.in/yaml.v3"
+)
+const (
+	frontmatterTagStart = "<frontmatter>"
+	frontmatterTagEnd   = "</frontmatter>"
+	bodyTagStart        = "<body>"
+	bodyTagEnd          = "</body>"
+)
+func processFileDoc(ctx context.Context, translator *PiTranslator, docsRoot, filePath, srcLang, tgtLang string, overwrite bool) (bool, error) {
+	absPath, relPath, err := resolveDocsPath(docsRoot, filePath)
+	if err != nil {
+		return false, err
+	}
+	content, err := os.ReadFile(absPath)
+	if err != nil {
+		return false, err
+	}
+	currentHash := hashBytes(content)
+	outputPath := filepath.Join(docsRoot, tgtLang, relPath)
+	if !overwrite {
+		skip, err := shouldSkipDoc(outputPath, currentHash)
+		if err != nil {
+			return false, err
+		}
+		if skip {
+			return true, nil
+		}
+	}
+	sourceFront, sourceBody := splitFrontMatter(string(content))
+	frontData := map[string]any{}
+	if strings.TrimSpace(sourceFront) != "" {
+		if err := yaml.Unmarshal([]byte(sourceFront), &frontData); err != nil {
+			return false, fmt.Errorf("frontmatter parse failed for %s: %w", relPath, err)
+		}
+	}
+	frontTemplate, markers := buildFrontmatterTemplate(frontData)
+	taggedInput := formatTaggedDocument(frontTemplate, sourceBody)
+	translatedDoc, err := translator.TranslateRaw(ctx, taggedInput, srcLang, tgtLang)
+	if err != nil {
+		return false, fmt.Errorf("translate failed (%s): %w", relPath, err)
+	}
+	translatedFront, translatedBody, err := parseTaggedDocument(translatedDoc)
+	if err != nil {
+		return false, fmt.Errorf("tagged output invalid for %s: %w", relPath, err)
+	}
+	if sourceFront != "" && strings.TrimSpace(translatedFront) == "" {
+		return false, fmt.Errorf("translation removed frontmatter for %s", relPath)
+	}
+	if err := applyFrontmatterTranslations(frontData, markers, translatedFront); err != nil {
+		return false, fmt.Errorf("frontmatter translation failed for %s: %w", relPath, err)
+	}
+	updatedFront, err := encodeFrontMatter(frontData, relPath, content)
+	if err != nil {
+		return false, err
+	}
+	if err := os.MkdirAll(filepath.Dir(outputPath), 0o755); err != nil {
+		return false, err
+	}
+	output := updatedFront + translatedBody
+	return false, os.WriteFile(outputPath, []byte(output), 0o644)
+}
+func formatTaggedDocument(frontMatter, body string) string {
+	return fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s", frontmatterTagStart, frontMatter, frontmatterTagEnd, bodyTagStart, body, bodyTagEnd)
+}
+func parseTaggedDocument(text string) (string, string, error) {
+	frontStart := strings.Index(text, frontmatterTagStart)
+	if frontStart == -1 {
+		return "", "", fmt.Errorf("missing %s", frontmatterTagStart)
+	}
+	frontStart += len(frontmatterTagStart)
+	frontEnd := strings.Index(text[frontStart:], frontmatterTagEnd)
+	if frontEnd == -1 {
+		return "", "", fmt.Errorf("missing %s", frontmatterTagEnd)
+	}
+	frontEnd += frontStart
+	bodyStart := strings.Index(text[frontEnd:], bodyTagStart)
+	if bodyStart == -1 {
+		return "", "", fmt.Errorf("missing %s", bodyTagStart)
+	}
+	bodyStart += frontEnd + len(bodyTagStart)
+	bodyEnd := strings.Index(text[bodyStart:], bodyTagEnd)
+	if bodyEnd == -1 {
+		return "", "", fmt.Errorf("missing %s", bodyTagEnd)
+	}
+	bodyEnd += bodyStart
+	prefix := strings.TrimSpace(text[:frontStart-len(frontmatterTagStart)])
+	suffix := strings.TrimSpace(text[bodyEnd+len(bodyTagEnd):])
+	if prefix != "" || suffix != "" {
+		return "", "", fmt.Errorf("unexpected text outside tagged sections")
+	}
+	frontMatter := trimTagNewlines(text[frontStart:frontEnd])
+	body := trimTagNewlines(text[bodyStart:bodyEnd])
+	return frontMatter, body, nil
+}
+func trimTagNewlines(value string) string {
+	value = strings.TrimPrefix(value, "\n")
+	value = strings.TrimSuffix(value, "\n")
+	return value
+}
+type frontmatterMarker struct {
+	Field string
+	Index int
+	Start string
+	End   string
+}
+func buildFrontmatterTemplate(data map[string]any) (string, []frontmatterMarker) {
+	if len(data) == 0 {
+		return "", nil
+	}
+	markers := []frontmatterMarker{}
+	lines := []string{}
+	if summary, ok := data["summary"].(string); ok {
+		start, end := markerPair("SUMMARY", 0)
+		markers = append(markers, frontmatterMarker{Field: "summary", Index: 0, Start: start, End: end})
+		lines = append(lines, fmt.Sprintf("summary: %s%s%s", start, summary, end))
+	}
+	if title, ok := data["title"].(string); ok {
+		start, end := markerPair("TITLE", 0)
+		markers = append(markers, frontmatterMarker{Field: "title", Index: 0, Start: start, End: end})
+		lines = append(lines, fmt.Sprintf("title: %s%s%s", start, title, end))
+	}
+	if readWhen, ok := data["read_when"].([]any); ok {
+		lines = append(lines, "read_when:")
+		for idx, item := range readWhen {
+			textValue, ok := item.(string)
+			if !ok {
+				lines = append(lines, fmt.Sprintf("  - %v", item))
+				continue
+			}
+			start, end := markerPair("READ_WHEN", idx)
+			markers = append(markers, frontmatterMarker{Field: "read_when", Index: idx, Start: start, End: end})
+			lines = append(lines, fmt.Sprintf("  - %s%s%s", start, textValue, end))
+		}
+	}
+	return strings.Join(lines, "\n"), markers
+}
+func markerPair(field string, index int) (string, string) {
+	return fmt.Sprintf("[[[FM_%s_%d_START]]]", field, index), fmt.Sprintf("[[[FM_%s_%d_END]]]", field, index)
+}
+func applyFrontmatterTranslations(data map[string]any, markers []frontmatterMarker, translatedFront string) error {
+	if len(markers) == 0 {
+		return nil
+	}
+	for _, marker := range markers {
+		value, err := extractMarkerValue(translatedFront, marker.Start, marker.End)
+		if err != nil {
+			return err
+		}
+		value = strings.TrimSpace(value)
+		switch marker.Field {
+		case "summary":
+			data["summary"] = value
+		case "title":
+			data["title"] = value
+		case "read_when":
+			data["read_when"] = setReadWhenValue(data["read_when"], marker.Index, value)
+		}
+	}
+	return nil
+}
+func extractMarkerValue(text, start, end string) (string, error) {
+	startIndex := strings.Index(text, start)
+	if startIndex == -1 {
+		return "", fmt.Errorf("missing marker %s", start)
+	}
+	startIndex += len(start)
+	endIndex := strings.Index(text[startIndex:], end)
+	if endIndex == -1 {
+		return "", fmt.Errorf("missing marker %s", end)
+	}
+	endIndex += startIndex
+	return text[startIndex:endIndex], nil
+}
+func setReadWhenValue(existing any, index int, value string) []any {
+	readWhen, ok := existing.([]any)
+	if !ok {
+		readWhen = []any{}
+	}
+	for len(readWhen) <= index {
+		readWhen = append(readWhen, "")
+	}
+	readWhen[index] = value
+	return readWhen
+}
+func shouldSkipDoc(outputPath string, sourceHash string) (bool, error) {
+	data, err := os.ReadFile(outputPath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return false, nil
+		}
+		return false, err
+	}
+	frontMatter, _ := splitFrontMatter(string(data))
+	if frontMatter == "" {
+		return false, nil
+	}
+	frontData := map[string]any{}
+	if err := yaml.Unmarshal([]byte(frontMatter), &frontData); err != nil {
+		return false, nil
+	}
+	storedHash := extractSourceHash(frontData)
+	if storedHash == "" {
+		return false, nil
+	}
+	return strings.EqualFold(storedHash, sourceHash), nil
+}
+func extractSourceHash(frontData map[string]any) string {
+	xi, ok := frontData["x-i18n"].(map[string]any)
+	if !ok {
+		return ""
+	}
+	value, ok := xi["source_hash"].(string)
+	if !ok {
+		return ""
+	}
+	return strings.TrimSpace(value)
+}
+func resolveDocsPath(docsRoot, filePath string) (string, string, error) {
+	absPath, err := filepath.Abs(filePath)
+	if err != nil {
+		return "", "", err
+	}
+	relPath, err := filepath.Rel(docsRoot, absPath)
+	if err != nil {
+		return "", "", err
+	}
+	if relPath == "." || relPath == "" {
+		return "", "", fmt.Errorf("file %s resolves to docs root %s", absPath, docsRoot)
+	}
+	if filepath.IsAbs(relPath) || relPath == ".." || strings.HasPrefix(relPath, ".."+string(filepath.Separator)) {
+		return "", "", fmt.Errorf("file %s not under docs root %s", absPath, docsRoot)
+	}
+	return absPath, relPath, nil
+}

package/scripts/docs-i18n/glossary.go ADDED Viewed

@@ -0,0 +1,29 @@
+package main
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+)
+type GlossaryEntry struct {
+	Source string `json:"source"`
+	Target string `json:"target"`
+}
+func LoadGlossary(path string) ([]GlossaryEntry, error) {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			return nil, nil
+		}
+		return nil, err
+	}
+	var entries []GlossaryEntry
+	if err := json.Unmarshal(data, &entries); err != nil {
+		return nil, fmt.Errorf("glossary parse failed: %w", err)
+	}
+	return entries, nil
+}

package/scripts/docs-i18n/go.mod ADDED Viewed

@@ -0,0 +1,10 @@
+module github.com/openclaw/openclaw/scripts/docs-i18n
+go 1.24.0
+require (
+	github.com/joshp123/pi-golang v0.0.4
+	github.com/yuin/goldmark v1.7.8
+	golang.org/x/net v0.50.0
+	gopkg.in/yaml.v3 v3.0.1
+)

package/scripts/docs-i18n/go.sum ADDED Viewed

@@ -0,0 +1,10 @@
+github.com/joshp123/pi-golang v0.0.4 h1:82HISyKNN8bIl2lvAd65462LVCQIsjhaUFQxyQgg5Xk=
+github.com/joshp123/pi-golang v0.0.4/go.mod h1:9mHEQkeJELYzubXU3b86/T8yedI/iAOKx0Tz0c41qes=
+github.com/yuin/goldmark v1.7.8 h1:iERMLn0/QJeHFhxSt3p6PeN9mGnvIKSpG9YYorDMnic=
+github.com/yuin/goldmark v1.7.8/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E=
+golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60=
+golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

package/scripts/docs-i18n/html_translate.go ADDED Viewed

@@ -0,0 +1,160 @@
+package main
+import (
+	"context"
+	"io"
+	"strings"
+	"github.com/yuin/goldmark"
+	"github.com/yuin/goldmark/ast"
+	"github.com/yuin/goldmark/extension"
+	"github.com/yuin/goldmark/text"
+	"golang.org/x/net/html"
+	"sort"
+)
+type htmlReplacement struct {
+	Start int
+	Stop  int
+	Value string
+}
+func translateHTMLBlocks(ctx context.Context, translator *PiTranslator, body, srcLang, tgtLang string) (string, error) {
+	source := []byte(body)
+	r := text.NewReader(source)
+	md := goldmark.New(
+		goldmark.WithExtensions(extension.GFM),
+	)
+	doc := md.Parser().Parse(r)
+	replacements := make([]htmlReplacement, 0, 8)
+	_ = ast.Walk(doc, func(n ast.Node, entering bool) (ast.WalkStatus, error) {
+		if !entering {
+			return ast.WalkContinue, nil
+		}
+		block, ok := n.(*ast.HTMLBlock)
+		if !ok {
+			return ast.WalkContinue, nil
+		}
+		start, stop, ok := htmlBlockSpan(block, source)
+		if !ok {
+			return ast.WalkSkipChildren, nil
+		}
+		htmlText := string(source[start:stop])
+		translated, err := translateHTMLBlock(ctx, translator, htmlText, srcLang, tgtLang)
+		if err != nil {
+			return ast.WalkStop, err
+		}
+		replacements = append(replacements, htmlReplacement{Start: start, Stop: stop, Value: translated})
+		return ast.WalkSkipChildren, nil
+	})
+	if len(replacements) == 0 {
+		return body, nil
+	}
+	return applyHTMLReplacements(body, replacements), nil
+}
+func htmlBlockSpan(block *ast.HTMLBlock, source []byte) (int, int, bool) {
+	lines := block.Lines()
+	if lines.Len() == 0 {
+		return 0, 0, false
+	}
+	start := lines.At(0).Start
+	stop := lines.At(lines.Len() - 1).Stop
+	if start >= stop {
+		return 0, 0, false
+	}
+	return start, stop, true
+}
+func applyHTMLReplacements(body string, replacements []htmlReplacement) string {
+	if len(replacements) == 0 {
+		return body
+	}
+	sortHTMLReplacements(replacements)
+	var out strings.Builder
+	last := 0
+	for _, rep := range replacements {
+		if rep.Start < last {
+			continue
+		}
+		out.WriteString(body[last:rep.Start])
+		out.WriteString(rep.Value)
+		last = rep.Stop
+	}
+	out.WriteString(body[last:])
+	return out.String()
+}
+func sortHTMLReplacements(replacements []htmlReplacement) {
+	sort.Slice(replacements, func(i, j int) bool {
+		return replacements[i].Start < replacements[j].Start
+	})
+}
+func translateHTMLBlock(ctx context.Context, translator *PiTranslator, htmlText, srcLang, tgtLang string) (string, error) {
+	tokenizer := html.NewTokenizer(strings.NewReader(htmlText))
+	var out strings.Builder
+	skipDepth := 0
+	for {
+		tt := tokenizer.Next()
+		if tt == html.ErrorToken {
+			if err := tokenizer.Err(); err != nil && err != io.EOF {
+				return "", err
+			}
+			break
+		}
+		raw := string(tokenizer.Raw())
+		tok := tokenizer.Token()
+		switch tt {
+		case html.StartTagToken:
+			out.WriteString(raw)
+			if isSkipTag(strings.ToLower(tok.Data)) {
+				skipDepth++
+			}
+		case html.EndTagToken:
+			out.WriteString(raw)
+			if isSkipTag(strings.ToLower(tok.Data)) && skipDepth > 0 {
+				skipDepth--
+			}
+		case html.SelfClosingTagToken:
+			out.WriteString(raw)
+		case html.TextToken:
+			if shouldTranslateHTMLText(skipDepth, raw) {
+				translated, err := translator.Translate(ctx, raw, srcLang, tgtLang)
+				if err != nil {
+					return "", err
+				}
+				out.WriteString(translated)
+			} else {
+				out.WriteString(raw)
+			}
+		default:
+			out.WriteString(raw)
+		}
+	}
+	return out.String(), nil
+}
+func shouldTranslateHTMLText(skipDepth int, text string) bool {
+	if strings.TrimSpace(text) == "" {
+		return false
+	}
+	return skipDepth == 0
+}
+func isSkipTag(tag string) bool {
+	switch tag {
+	case "code", "pre", "script", "style":
+		return true
+	default:
+		return false
+	}
+}