glotstack 0.0.6 → 0.0.8

package/package.json

@@ -1,36 +1,41 @@
 {
   "name": "glotstack",
-  "version": "0.0.6",
+  "version": "0.0.8",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "author": "JD Cumpson",
   "license": "MIT",
   "private": false,
   "dependencies": {
+    "@eslint/js": "^9.26.0",
     "commander": "^13.1.0",
-    "eslint": "^9.26.0"
+    "eslint": "^9.26.0",
+    "eslint-plugin-i18next": "^6.1.1",
+    "eslint-plugin-react": "^7.37.5",
+    "form-data": "^4.0.2",
+    "undici": "^7.9.0"
   },
   "devDependencies": {
-    "@eslint/js": "^9.26.0",
     "@types/js-yaml": "^4.0.9",
     "@types/node": "^22.15.17",
-    "@types/react": "^19.1.4",
+    "@types/react": "^18.3.1",
+    "@types/react-dom": "^18.3.1",
     "globals": "^16.1.0",
     "js-yaml": "^4.1.0",
+    "nodemon": "^3.1.10",
     "typescript": "5.4.4",
     "typescript-eslint": "^8.32.1"
   },
   "peerDependencies": {
-    "react": "^19.1.0"
-  },
-  "resolutions": {
-    "react": "18.3.1"
+    "react": "^18.3.1",
+    "react-dom": "^18.3.1"
   },
   "bin": {
     "glotstack": "dist/cli.js"
   },
   "scripts": {
     "build": "tsc && scripts/fix-shebang.sh dist/cli.js",
+    "watch": "nodemon --watch src --ext ts,tsx,mjs,json --exec \"bash -c 'npm run build'\"",
     "prepublishOnly": "yarn run build",
     "glotstack": "node dist/cli.js"
   }

package/src/cli.tsx

@@ -1,6 +1,6 @@
 import { Command, program } from 'commander'
 import path from 'path'
-import { promises as fs, read } from 'fs'
+import { promises as fs, createReadStream } from 'fs'
 import { findGlotstackConfig } from './util/findConfig'
 import { cwd } from 'process'
 import { merge } from './util/object'
@@ -8,36 +8,102 @@ import { loadYaml } from './util/yaml'
 import eslint from 'eslint'
 import * as readline from "node:readline/promises"
 import { stdin as input, stdout as output } from "node:process"
+import { Translations } from 'src'
+import { fetch } from 'undici'
+import { FormData } from 'undici'
+import { openAsBlob } from 'node:fs'
+import { readdir, readFile, writeFile } from 'node:fs/promises'
+import { resolve } from 'node:path'
+
+const fetchGlotstack = async function <T>(url: string, apiKey: string, body: Record<string, any> | FormData, overrideHeaders?: Record<string, any>): Promise<T> {
+  console.info(`Fetching glotstack.ai: ${url}`)
+
+  const headers: Record<string, any> = {
+    'authorization': `Bearer ${apiKey}`,
+    ...(overrideHeaders == null ? {} : overrideHeaders),
+  }
 
+  let payloadBody: FormData | string
 
-const fetchGlotstack = async (apiOrigin: string, apiKey: string, body: object) => {
-  const url = `${apiOrigin}/api/translations`
-  console.info(`Fetching translations from: ${url}`)
-
-  const headers = {
-    'Content-Type': 'application/json',
-    'Authorization': `Bearer ${apiKey}`,
+  if (!(body instanceof FormData)) {
+    headers['content-type'] = 'application/json'
+    payloadBody = JSON.stringify(body)
+  } else {
+    payloadBody = body
   }
 
   try {
-    const res = await fetch(url, { method: 'POST', body: JSON.stringify(body), headers })
+    const res = await fetch(url, { method: 'POST', body: payloadBody, headers })
     if (!res.ok) throw new Error(`HTTP ${res.status}: ${res.statusText}`)
-    return res.json()
-    // fs.writeFile(`${outputDir}/source.json`, JSON.stringify(json, null, 2))
+    return res.json() as T
   } catch (err) {
     console.error('Fetch failed:', err)
     process.exit(1)
   }
 }
 
+function unflatten(flat: Record<string, any>): Record<string, any> {
+  const result: Record<string, any> = {}
+
+  for (const flatKey in flat) {
+    const parts = flatKey.split('.')
+    let current = result
+
+    parts.forEach((part, idx) => {
+      if (idx === parts.length - 1) {
+        current[part] = flat[flatKey]
+      } else {
+        if (!(part in current)) {
+          current[part] = {}
+        }
+        current = current[part]
+      }
+    })
+  }
+
+  return result
+}
+
+const DEFAULT_OPTIONS: Record<string, string> = {
+  sourcePath: '.',
+  sourceLocale: 'en-US',
+  apiOrigin: 'https://glotstack.ai'
+}
+
+async function resolveConfigAndOptions(options: Record<string, any>) {
+
+  const config = await findGlotstackConfig(cwd()) ?? {}
+
+  if ('outputLocales' in options) {
+    if ((options.outputLocales as string[]).includes('en-US')) {
+      console.warn('en-US detected in outputLocales, removing')
+      options.outputLocales = options.outputLocales.filter((x: string) => x !== 'en-US')
+    }
+  }
+
+  const resolved = merge({}, DEFAULT_OPTIONS, config, options)
+
+  // special case to match source
+  if (resolved.outputDir == null) {
+    resolved.outputDir = resolved.sourcePath
+  }
+
+  return resolved
+}
+
 
 async function run(args: string[]) {
   program
     .command('extract-translations')
-    .option('--source-path [path]', 'to source files root directory', '.')
-    .option('--api-origin [url]', 'glotstack api origin', process.env.GLOTSTACK_HOST ?? 'https://glotstack.ai')
-    .option('--yes', 'skip confirm checks', false)
-    .action(async (options: Record<string, any>) => {
+    .description('extract translations from all compatible source files.')
+    .option('--source-path [path]', `path directory containing [locale].json files (default=${DEFAULT_OPTIONS['sourcePath']})`)
+    .option('--source-locale [locale]', `the locale you provide "context" in, your primary locale (default=${DEFAULT_OPTIONS['sourceLocale']})`)
+    .option('--api-origin [url]', `glotstack api origin (default=${DEFAULT_OPTIONS['apiOrigin']})`)
+    .option('--output-dir [path]', 'path to output directory (default=<source-path>')
+    .option('--api-key [key]', 'api key for glotstack.ai')
+    .option('--yes', 'skip confirm checks')
+    .action(async (inputOptions: Record<string, any>) => {
+      const options = await resolveConfigAndOptions(inputOptions)
       if (!options.apiOrigin) {
         throw new Error('apiOrigin must be specified')
       }
@@ -49,7 +115,6 @@ async function run(args: string[]) {
         .map((r) => r.filePath)
 
       const rl = readline.createInterface({ input, output })
-
       const askToSend = async (): Promise<boolean> => {
         if (options.yes) {
           return true
@@ -67,7 +132,17 @@ async function run(args: string[]) {
 
       const send = await askToSend()
       if (send) {
-        console.info('Sending files to LLM')
+        console.info('Sending files to generate new source and extracted strings')
+        const url = `${options.apiOrigin}/uploads/translations/extract`
+        const form = new FormData()
+
+        for (let i = 0; i < filesWithIssues.length; i++) {
+          const filePath = filesWithIssues[i]
+          form.append(`file_${i}`, await openAsBlob(filePath), filePath)
+          console.debug(`Uploading file: ${filePath}`)
+        }
+        const data = await fetchGlotstack<{ translations: { name: string; modified_source: { url: string } }[] }>(url, options.apiKey, form)
+        data.translations.map(elem => console.info(`Source and translations available for: ${elem.name}:\n  ${elem.modified_source.url}\n\n`))
         rl.close()
       } else {
         rl.close()
@@ -76,46 +151,27 @@ async function run(args: string[]) {
 
   program
     .command('get-translations')
-    .option('--source-path [path]', 'path to en-US.json (or your canonical source json)')
-    .option('--api-origin [url]', 'glotstack api origin', process.env.GLOTSTACK_HOST ?? 'https://glotstack.ai')
-    .option('--output-dir [path]', 'path to output directory')
+    .description('fetch translations for all [output-locals...]. Use .glotstack.json for repeatable results.')
+    .option('--source-path [path]', `path directory containing [locale].json files (default=${DEFAULT_OPTIONS['sourcePath']})`)
+    .option('--source-locale [locale]', `the locale you provide "context" in, your primary locale (default=${DEFAULT_OPTIONS['sourceLocale']})`)
+    .option('--api-origin [url]', `glotstack api origin (default=${DEFAULT_OPTIONS['apiOrigin']})`)
+    .option('--output-dir [path]', 'path to output directory (default=<source-path>')
     .option('--api-key [key]', 'api key for glotstack.ai')
     .option('--project-id [id]', '(optional) specific project to use')
     .argument('[output-locales...]', 'locales to get translations for')
     .action(async (outputLocales: string[], options: Record<string, any>, command: Command) => {
-      const configPath = findGlotstackConfig(cwd())
-      let config = {}
-
-      if (configPath != null) {
-        console.info('Loading config file at ', configPath)
-        try {
-          const text = await fs.readFile(configPath, 'utf-8')
-          config = JSON.parse(text)
-          console.info('Loaded config file', config)
-        } catch (err) {
-          //pass
-        }
-      }
-
-      const resolved = merge(config, options, { outputLocales })
-      const { apiOrigin, sourcePath, outputDir, projectId } = resolved
-
-      if ((resolved.outputLocales as string[]).includes('en-US')) {
-        console.warn('en-US detected in outputLocales, removing')
-        resolved.outputLocales = resolved.outputLocales.filter((x: string) => x !== 'en-US')
-      }
-
-      if (!sourcePath) {
+      const resolved = await resolveConfigAndOptions({ ...options, outputLocales: outputLocales })
+      if (!resolved.sourcePath) {
         throw new Error('sourcePath must be specified')
       }
-      if (!apiOrigin) {
+      if (!resolved.apiOrigin) {
         throw new Error('apiOrigin must be specified')
       }
-      if (!outputDir) {
+      if (!resolved.outputDir) {
         throw new Error('outputDir must be specified')
       }
 
-      const absPath = path.resolve(sourcePath)
+      const absPath = path.resolve(resolved.sourcePath, `${resolved.sourceLocale}.json`)
       const fileContent = await fs.readFile(absPath, 'utf-8')
 
       let json = null
@@ -133,18 +189,62 @@ async function run(args: string[]) {
       const body = {
         locales: resolved.outputLocales,
         translations: json,
-        ...{ ... (projectId != null ? { projectId } : {}) },
+        ...{ ... (resolved.projectId != null ? { projectId: resolved.projectId } : {}) },
       }
 
-      const data = await fetchGlotstack(apiOrigin, resolved.apiKey, body)
+      const url = `${resolved.apiOrigin}/api/translations`
+      const data = await fetchGlotstack<{ data: Translations }>(url, resolved.apiKey, body)
       console.info('Received translations:', data)
       Object.entries(data.data).map(([key, val]) => {
-        const p = `${outputDir}/${key}.json`
+        const p = `${resolved.outputDir}/${key}.json`
         console.info(`Writing file ${p}`)
-        fs.writeFile(`${outputDir}/${key}.json`, JSON.stringify(val, null, 2))
+        fs.writeFile(`${resolved.outputDir}/${key}.json`, JSON.stringify(val, null, 2))
       })
     })
 
+  program
+    .command('format-json')
+    .description('format files in --source-path [path] to nested (not flat)')
+    .option('--source-path [path]', `path directory containing [locale].json files (default=${DEFAULT_OPTIONS['sourcePath']})`)
+    .option('--yes', 'skip confirm checks')
+    .action(async (inputOptions: Record<string, any>) => {
+      const options = await resolveConfigAndOptions(inputOptions)
+
+      if (!options.sourcePath) {
+        throw new Error('sourcePath must be specified')
+      }
+      const rl = readline.createInterface({ input, output })
+      const askToSend = async (): Promise<boolean> => {
+        if (options.yes) {
+          return true
+        }
+        const response = await rl.question(`This will update your source files -- have you checked them into SCM/git? Type yes to proceed (yes/no):`)
+        if (response === 'yes') {
+          return true
+        } else if (response !== 'no') {
+          console.error('Please respond with yes or no.')
+          return askToSend()
+        } else {
+          return false
+        }
+      }
+      const yes = await askToSend()
+      if (yes) {
+        const files = await readdir(options.sourcePath)
+        for (let i = 0; i < (await files).length; i++) {
+          const fp = resolve(options.sourcePath, files[i])
+          const text = await readFile(fp, 'utf-8')
+          const json = JSON.parse(text)
+          const formatted = JSON.stringify(unflatten(json), null, 2)
+          await writeFile(fp, formatted)
+        }
+        rl.close()
+      }
+      rl.close()
+
+    })
+
+
   await program.parseAsync(args)
 }
 

package/src/index.tsx

@@ -1,15 +1,8 @@
 import * as React from 'react'
-import { merge, isEqual } from './util/object'
+import { merge } from './util/object'
+import { debug } from 'node:console'
 
-export type LocaleRegion =
-  | 'en'
-  | 'en-US'
-  | 'en-US-genz'
-  | 'fr-FR'
-  | 'de-DE'
-  | 'nl-NL'
-  | 'jp-JP'
-  | string
+export type LocaleRegion = string
 
 
 export interface TranslationLeaf {
@@ -21,13 +14,15 @@ export interface Translations {
   [key: string]: Translations | TranslationLeaf
 }
 
+
 export interface ContextType {
   translations: Translations
   locale: string | null
   loadTranslations: (locale: LocaleRegion) => Promise<Translations>
   setLocale: (locale: LocaleRegion) => void
-  importMethod: (locale: LocaleRegion) => Promise<Translations>
+  importMethod: (locale: LocaleRegion) => Promise<Translations> 
   t: (key: string, options?: { locale?: LocaleRegion }) => string
+  ssr: boolean
 }
 
 export const GlotstackContext = React.createContext<ContextType>({
@@ -37,18 +32,61 @@ export const GlotstackContext = React.createContext<ContextType>({
   locale: null,
   importMethod: (_locale: LocaleRegion) => { throw new Error('import method not set') },
   t: () => { throw new Error('import method not set') },
+  ssr: false
 })
 
 interface GlotstackProviderProps {
   children: React.ReactNode
-  initialTranslations?: Translations
+  initialTranslations?: Record<string, Translations>
   initialLocale?: LocaleRegion
   onTranslationLoaded?: (locale: LocaleRegion, translations: Translations) => void
   onLocaleChange?: (locale: LocaleRegion) => void
-  importMethod: (locale: LocaleRegion) => Promise<Translations>
+  importMethod: ContextType['importMethod']
+  ssr?: boolean
+}
+
+export enum LogLevel {
+  DEBUG = 0,
+  LOG = 1,
+  INFO = 2,
+  WARNING = 3,
+  ERROR = 4,
+}
+
+const LogLevelToFunc: Record<LogLevel, (...args: Parameters<typeof console.info>) => void> = {
+  [LogLevel.DEBUG]: console.debug,
+  [LogLevel.INFO]: console.info,
+  [LogLevel.LOG]: console.log,
+  [LogLevel.WARNING]: console.warn,
+  [LogLevel.ERROR]: console.error,
+} as const
+
+let logLevel: LogLevel = LogLevel.DEBUG
+
+export const setLogLevel = (level: LogLevel) => {
+  logLevel = level
+}
+
+const makeLoggingFunction = (level: LogLevel) => (...args: Parameters<typeof console.info>) => {
+  const func = LogLevelToFunc[level]
+  if (level < logLevel) {
+    return
+  }
+  return func(`[level=${level} logLevel=${logLevel}][glotstack.ai]`, ...args)
+}
+
+const logger = {
+  debug: makeLoggingFunction(LogLevel.DEBUG),
+  info: makeLoggingFunction(LogLevel.INFO),
+  warn: makeLoggingFunction(LogLevel.WARNING),
+  error: makeLoggingFunction(LogLevel.ERROR),
+
 }
 
 export const access = (key: string, locale: LocaleRegion, translations: Translations) => {
+  if (translations == null) {
+    return key
+  }
   const access = [...key.split('.')] as [LocaleRegion, ...string[]]
   const localeTranslations = translations?.[locale]
 
@@ -64,21 +102,29 @@ export const access = (key: string, locale: LocaleRegion, translations: Translat
   return (value?.value ?? key) as string
 }
 
-export const GlotstackProvider = ({ children, initialLocale, initialTranslations, onLocaleChange, onTranslationLoaded, importMethod }: GlotstackProviderProps) => {
+export const GlotstackProvider = ({ children, initialLocale, initialTranslations, onLocaleChange, onTranslationLoaded, importMethod, ssr}: GlotstackProviderProps) => {
   if (initialLocale == null) {
     throw new Error('initialLocale must be set')
   }
   const [locale, setLocale] = React.useState<LocaleRegion>(initialLocale)
-  const translationsRef = React.useRef(initialTranslations)
+  const translationsRef = React.useRef<Record<string, Translations>|null>(initialTranslations || null)
   const loadingRef = React.useRef<Record<string, Promise<Translations>>>({})
 
-  const loadTranslations = React.useCallback(async (locale: string) => {
+  const loadTranslations = React.useCallback(async (locale: string, opts?: {force?: boolean}) => {
     // TODO: if translations are loaded only reload if some condition is
     try {
-      if (loadingRef.current?.[locale] != null) {
+      if (loadingRef.current?.[locale] != null && opts?.force != true) {
+        logger.debug('Waiting for translations already loading', locale)
         return (await loadingRef.current?.[locale])
       }
-      loadingRef.current[locale] = importMethod(locale)
+      if (translationsRef.current?.[locale] != null && opts?.force != true) {
+        logger.debug('Skipping load for translations', locale, translationsRef.current?.[locale], translationsRef.current)
+        return translationsRef.current?.[locale]
+      }
+      if (loadingRef.current != null) {
+        logger.debug('Loading translations', locale, merge({}, translationsRef.current ?? {}))
+        loadingRef.current[locale] = importMethod(locale)
+      }
       const result = await loadingRef.current[locale]
 
       if (result == null) {
@@ -90,7 +136,7 @@ export const GlotstackProvider = ({ children, initialLocale, initialTranslations
       onTranslationLoaded?.(locale, result)
       return result
     } catch (err) {
-      console.error('Unable to import translations', err)
+      logger.error('Unable to import translations', err)
       throw err
     }
   }, [importMethod, onTranslationLoaded])
@@ -98,7 +144,7 @@ export const GlotstackProvider = ({ children, initialLocale, initialTranslations
   React.useEffect(() => {
     const run = async () => {
       onLocaleChange?.(locale)
-      const result = await loadTranslations(locale)
+      await loadTranslations(locale)
     }
     React.startTransition(() => {
       run()
@@ -113,15 +159,11 @@ export const GlotstackProvider = ({ children, initialLocale, initialTranslations
       importMethod,
       loadTranslations,
       t: (key: string, opts?: { locale?: LocaleRegion }) => {
-        React.useEffect(() => {
-          if (opts?.locale == null) {
-            return
-          }
-          loadTranslations(opts?.locale)
-        }, [locale, opts?.locale])
+        const resolvedLocale = opts?.locale ?? locale
+        loadTranslations(resolvedLocale)
         return access(key, opts?.locale ?? locale, translationsRef.current ?? {})
-      }
-
+      },
+      ssr: ssr == true,
     }
   }, [locale, importMethod])
 
@@ -134,7 +176,6 @@ export const useGlotstack = () => {
   return React.useContext(GlotstackContext)
 }
 
-
 export const useTranslations = (_options?: Record<never, never>) => {
   const context = React.useContext(GlotstackContext)
   return context

package/src/util/findConfig.ts

@@ -1,26 +1,42 @@
-import * as fs from 'fs';
-import * as path from 'path';
+import { existsSync } from 'fs'
+import { readFile } from 'fs/promises'
+import * as path from 'path'
 
 /**
  * Recursively looks for `.glotstack.json` from the current directory up to the root.
  * @param startDir The directory to start the search from. Defaults to process.cwd().
  * @returns The absolute path to the file if found, or null if not found.
  */
-export function findGlotstackConfig(startDir: string = process.cwd()): string | null {
-  let currentDir = path.resolve(startDir);
+export async function findGlotstackConfig(startDir: string = process.cwd()): Promise<object | null> {
+  let currentDir = path.resolve(startDir)
+  let configPath = null
 
   while (true) {
-    const candidate = path.join(currentDir, '.glotstack.json');
-    if (fs.existsSync(candidate)) {
-      return candidate;
+    const candidate = path.join(currentDir, '.glotstack.json')
+    if (existsSync(candidate)) {
+      configPath = candidate
     }
 
-    const parentDir = path.dirname(currentDir);
+    const parentDir = path.dirname(currentDir)
     if (parentDir === currentDir) {
-      break; // Reached root
+      break // Reached root
     }
-    currentDir = parentDir;
+    currentDir = parentDir
   }
 
-  return null;
+  let config = {}
+
+  if (configPath != null) {
+    console.info('Loading config file at ', configPath)
+    try {
+      const text = await readFile(configPath, 'utf-8')
+      config = JSON.parse(text)
+      console.info('Loaded config file', config)
+      return config
+    } catch (err) {
+      console.warn('Could not load config', configPath)
+    }
+  }
+  console.warn('Could not find any .glotstack.json config files')
+  return null
 }

package/tsconfig.json

@@ -8,7 +8,7 @@
     "module": "CommonJS",
     "declaration": true,
     "strict": true,
-    "jsx": "react",
+    "jsx": "react-jsx",
     "sourceMap": true,
     "baseUrl": ".",
     "paths": {