@blockdia/scratch-blocks 0.1.0

package/i18n/js_to_json.py

@@ -0,0 +1,136 @@
+#!/usr/bin/python
+
+# Gives the translation status of the specified apps and languages.
+#
+# Copyright 2013 Google Inc.
+# https://developers.google.com/blockly/
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Extracts messages from .js files into .json files for translation.
+
+Specifically, lines with the following formats are extracted:
+
+    /// Here is a description of the following message.
+    Blockly.SOME_KEY = 'Some value';
+
+Adjacent "///" lines are concatenated.
+
+There are two output files, each of which is proper JSON.  For each key, the
+file en.json would get an entry of the form:
+
+    "Blockly.SOME_KEY", "Some value",
+
+The file qqq.json would get:
+
+    "Blockly.SOME_KEY", "Here is a description of the following message.",
+
+Commas would of course be omitted for the final entry of each value.
+
+@author Ellen Spertus (ellen.spertus@gmail.com)
+"""
+
+import argparse
+import codecs
+import json
+import os
+import re
+from common import write_files
+
+
+_INPUT_DEF_PATTERN = re.compile("""Blockly.Msg.(\w*)\s*=\s*'(.*)';?\r?$""")
+
+_INPUT_SYN_PATTERN = re.compile(
+    """Blockly.Msg.(\w*)\s*=\s*Blockly.Msg.(\w*);""")
+
+_CONSTANT_DESCRIPTION_PATTERN = re.compile(
+    """{{Notranslate}}""", re.IGNORECASE)
+
+def main():
+  # Set up argument parser.
+  parser = argparse.ArgumentParser(description='Create translation files.')
+  parser.add_argument(
+      '--author',
+      default='Ellen Spertus <ellen.spertus@gmail.com>',
+      help='name and email address of contact for translators')
+  parser.add_argument('--lang', default='en',
+                      help='ISO 639-1 source language code')
+  parser.add_argument('--output_dir', default='json',
+                      help='relative directory for output files')
+  parser.add_argument('--input_file', default='messages.js',
+                      help='input file')
+  parser.add_argument('--quiet', action='store_true', default=False,
+                      help='only display warnings, not routine info')
+  args = parser.parse_args()
+  if (not args.output_dir.endswith(os.path.sep)):
+    args.output_dir += os.path.sep
+
+  # Read and parse input file.
+  results = []
+  synonyms = {}
+  constants = {}  # Values that are constant across all languages.
+  description = ''
+  infile = codecs.open(args.input_file, 'r', 'utf-8')
+  for line in infile:
+    if line.startswith('///'):
+      if description:
+        description = description + ' ' + line[3:].strip()
+      else:
+        description = line[3:].strip()
+    else:
+      match = _INPUT_DEF_PATTERN.match(line)
+      if match:
+        key = match.group(1)
+        value = match.group(2).replace("\\'", "'")
+        if not description:
+          print('Warning: No description for ' + result['meaning'])
+        if (description and _CONSTANT_DESCRIPTION_PATTERN.search(description)):
+          constants[key] = value
+        else:
+          result = {}
+          result['meaning'] = key
+          result['source'] = value
+          result['description'] = description
+          results.append(result)
+        description = ''
+      else:
+        match = _INPUT_SYN_PATTERN.match(line)
+        if match:
+          if description:
+            print('Warning: Description preceding definition of synonym {0}.'.
+                  format(match.group(1)))
+            description = ''
+          synonyms[match.group(1)] = match.group(2)
+  infile.close()
+
+  # Create <lang_file>.json, keys.json, and qqq.json.
+  write_files(args.author, args.lang, args.output_dir, results, False)
+
+  # Create synonyms.json.
+  synonym_file_name = os.path.join(os.curdir, args.output_dir, 'synonyms.json')
+  with open(synonym_file_name, 'w') as outfile:
+    json.dump(synonyms, outfile)
+  if not args.quiet:
+    print("Wrote {0} synonym pairs to {1}.".format(
+        len(synonyms), synonym_file_name))
+
+  # Create constants.json
+  constants_file_name = os.path.join(os.curdir, args.output_dir, 'constants.json')
+  with open(constants_file_name, 'w') as outfile:
+    json.dump(constants, outfile)
+  if not args.quiet:
+    print("Wrote {0} constant pairs to {1}.".format(
+        len(constants), synonym_file_name))
+
+if __name__ == '__main__':
+  main()

package/i18n/json_to_js.js

@@ -0,0 +1,52 @@
+const async = require('async');
+const fs = require('fs');
+const path = require('path');
+const glob = require('glob');
+
+// Globals
+const PATH_INPUT = path.resolve(__dirname, '../msg/json/en.json');
+// If you want to generate js files for other languages, comment out the line above,
+// and use the one below instead.
+// const PATH_INPUT = path.resolve(__dirname, '../msg/json/*.json');
+const PATH_OUTPUT = path.resolve(__dirname, '../msg/js');
+const CONCURRENCY_LIMIT = 4;
+
+// Processing task
+const work = function (uri, callback) {
+    fs.readFile(uri, function (err, body) {
+      const name = path.parse(uri).name;
+      if (err) return callback(err);
+
+        // Convert file body into an object (let this throw if invalid JSON)
+        body = JSON.parse(body);
+
+        // File storage object and preamble
+        let file = '';
+        file += '// This file was automatically generated.  Do not modify.\n';
+        file += '\n';
+        file += '\'use strict\';\n';
+        file += '\n';
+        file += `goog.provide(\'Blockly.Msg.${name}\');\n`;
+        file += 'goog.require(\'Blockly.Msg\');\n';
+        file += '\n';
+
+        // Iterate over object and build up file
+        for (let i in body) {
+            file += `Blockly.Msg["${i}"] = "${body[i].replace(/"/g, '\\"')}";\n`
+        }
+
+        // Write file to disk
+        fs.writeFile(`${PATH_OUTPUT}/${name}.js`, file, callback);
+    });
+};
+
+// Create async processing queue
+const q = async.queue(work, CONCURRENCY_LIMIT);
+
+// Handle errors
+q.error = function (err) {
+    throw new Error(err);
+};
+
+// Get all JSON files in input directory and add to queue
+q.push(glob.sync(PATH_INPUT));

package/i18n/json_to_js.py

@@ -0,0 +1,185 @@
+#!/usr/bin/python
+
+# Converts .json files into .js files for use within Blockly apps.
+#
+# Copyright 2013 Google Inc.
+# https://developers.google.com/blockly/
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import codecs      # for codecs.open(..., 'utf-8')
+import glob
+import json        # for json.load()
+import os          # for os.path()
+import subprocess  # for subprocess.check_call()
+from common import InputError
+from common import read_json_file
+
+
+# Store parsed command-line arguments in global variable.
+args = None
+
+
+def _create_xlf(target_lang):
+    """Creates a <target_lang>.xlf file for Soy.
+
+    Args:
+        target_lang: The ISO 639 language code for the target language.
+            This is used in the name of the file and in the metadata.
+
+    Returns:
+        A pointer to a file to which the metadata has been written.
+
+    Raises:
+        IOError: An error occurred while opening or writing the file.
+    """
+    filename = os.path.join(os.curdir, args.output_dir, target_lang + '.xlf')
+    out_file = codecs.open(filename, 'w', 'utf-8')
+    out_file.write("""<?xml version="1.0" encoding="UTF-8"?>
+<xliff version="1.2" xmlns="urn:oasis:names:tc:xliff:document:1.2">
+  <file original="SoyMsgBundle"
+        datatype="x-soy-msg-bundle"
+        xml:space="preserve"
+        source-language="{0}"
+        target-language="{1}">
+    <body>""".format(args.source_lang, target_lang))
+    return out_file
+
+
+def _close_xlf(xlf_file):
+    """Closes a <target_lang>.xlf file created with create_xlf().
+
+    This includes writing the terminating XML.
+
+    Args:
+        xlf_file: A pointer to a file created by _create_xlf().
+
+    Raises:
+        IOError: An error occurred while writing to or closing the file.
+    """
+    xlf_file.write("""
+    </body>
+  </file>
+</xliff>
+""")
+    xlf_file.close()
+
+
+def _process_file(path_to_json, target_lang, key_dict):
+    """Creates an .xlf file corresponding to the specified .json input file.
+
+    The name of the input file must be target_lang followed by '.json'.
+    The name of the output file will be target_lang followed by '.js'.
+
+    Args:
+        path_to_json: Path to the directory of xx.json files.
+        target_lang: A IETF language code (RFC 4646), such as 'es' or 'pt-br'.
+        key_dict: Dictionary mapping Blockly keys (e.g., Maze.turnLeft) to
+            Closure keys (hash numbers).
+
+    Raises:
+        IOError: An I/O error occurred with an input or output file.
+        InputError: Input JSON could not be parsed.
+        KeyError: Key found in input file but not in key file.
+    """
+    keyfile = os.path.join(path_to_json, target_lang + '.json')
+    j = read_json_file(keyfile)
+    out_file = _create_xlf(target_lang)
+    for key in j:
+        if key != '@metadata':
+            try:
+                identifier = key_dict[key]
+            except KeyError as e:
+                print('Key "%s" is in %s but not in %s' %
+                      (key, keyfile, args.key_file))
+                raise e
+            target = j.get(key)
+            out_file.write(u"""
+      <trans-unit id="{0}" datatype="html">
+        <target>{1}</target>
+      </trans-unit>""".format(identifier, target))
+    _close_xlf(out_file)
+
+
+def main():
+    """Parses arguments and iterates over files."""
+
+    # Set up argument parser.
+    parser = argparse.ArgumentParser(description='Convert JSON files to JS.')
+    parser.add_argument('--source_lang', default='en',
+                        help='ISO 639-1 source language code')
+    parser.add_argument('--output_dir', default='generated',
+                        help='relative directory for output files')
+    parser.add_argument('--key_file', default='json' + os.path.sep + 'keys.json',
+                        help='relative path to input keys file')
+    parser.add_argument('--template', default='template.soy')
+    parser.add_argument('--path_to_jar',
+                        default='..' + os.path.sep + 'apps' + os.path.sep
+                        + '_soy',
+                        help='relative path from working directory to '
+                        'SoyToJsSrcCompiler.jar')
+    parser.add_argument('files', nargs='+', help='input files')
+
+    # Initialize global variables.
+    global args
+    args = parser.parse_args()
+
+    # Make sure output_dir ends with slash.
+    if (not args.output_dir.endswith(os.path.sep)):
+      args.output_dir += os.path.sep
+
+    # Read in keys.json, mapping descriptions (e.g., Maze.turnLeft) to
+    # Closure keys (long hash numbers).
+    key_file = open(args.key_file)
+    key_dict = json.load(key_file)
+    key_file.close()
+
+    # Process each input file.
+    print('Creating .xlf files...')
+    processed_langs = []
+    if len(args.files) == 1:
+      # Windows does not expand globs automatically.
+      args.files = glob.glob(args.files[0])
+    for arg_file in args.files:
+      (path_to_json, filename) = os.path.split(arg_file)
+      if not filename.endswith('.json'):
+        raise InputError(filename, 'filenames must end with ".json"')
+      target_lang = filename[:filename.index('.')]
+      if not target_lang in ('qqq', 'keys'):
+        processed_langs.append(target_lang)
+        _process_file(path_to_json, target_lang, key_dict)
+
+    # Output command line for Closure compiler.
+    if processed_langs:
+      print('Creating .js files...')
+      processed_lang_list = ','.join(processed_langs)
+      subprocess.check_call([
+          'java',
+          '-jar', os.path.join(args.path_to_jar, 'SoyToJsSrcCompiler.jar'),
+          '--locales', processed_lang_list,
+          '--messageFilePathFormat', args.output_dir + '{LOCALE}.xlf',
+          '--outputPathFormat', args.output_dir + '{LOCALE}.js',
+          '--srcs', args.template])
+      if len(processed_langs) == 1:
+        print('Created ' + processed_lang_list + '.js in ' + args.output_dir)
+      else:
+        print('Created {' + processed_lang_list + '}.js in ' + args.output_dir)
+
+      for lang in processed_langs:
+        os.remove(args.output_dir + lang + '.xlf')
+      print('Removed .xlf files.')
+
+
+if __name__ == '__main__':
+    main()

package/i18n/sync_tx_translations.js

@@ -0,0 +1,111 @@
+#!/usr/bin/env node
+
+/**
+ * @fileoverview
+ * Script to pull translations for blocks from transifex and generate the scratch_msgs file.
+ * Expects that the project and resource have already been defined in Transifex, and that
+ * the person running the script has the the TX_TOKEN environment variable set to an api
+ * token that has developer access.
+ */
+
+ const usage = `
+ Pull supported language translations from Transifex. Usage:
+   node sync_tx_translations.js
+   NOTE: TX_TOKEN environment variable needs to be set with a Transifex API token. See
+   the Localization page on the GUI wiki for information about setting up Transifex.
+ `;
+// Fail immediately if the TX_TOKEN is not defined
+if (!process.env.TX_TOKEN || process.argv.length !== 2) {
+  process.stdout.write(usage);
+  process.exit(1);
+};
+
+const fs = require('fs');
+const path = require('path');
+const assert = require('assert');
+const locales = require('scratch-l10n').default;
+const {txPull} = require('scratch-l10n/lib/transifex.js');
+
+// Globals
+const PATH_OUTPUT = path.resolve(__dirname, '../msg');
+const PROJECT = 'scratch-editor'
+const RESOURCE = 'blocks';
+const MODE = 'reviewed';
+
+
+
+let en = fs.readFileSync(path.resolve(__dirname, '../msg/json/en.json'));
+en = JSON.parse(en);
+const enKeys = Object.keys(en).sort().toString();
+
+// Check that translation is valid:
+// entry: array [key, translation]  corresponding to a single string from <locale>.json
+// - messages with placeholders have the same number of placeholders
+// - messages must not have newlines embedded
+const validateEntry = function (entry) {
+    const re = /(%\d)/g;
+    const [key, translation] = entry;
+    const enMatch = en[key].match(re);
+    const tMatch = translation.match(re);
+    const enCount = enMatch ? enMatch.length : 0;
+    const tCount = tMatch ? tMatch.length : 0;
+    assert.strictEqual(tCount, enCount, `${key}:${en[key]} - "${translation}" placeholder mismatch`);
+    if (enCount > 0) {
+
+      assert.notStrictEqual(tMatch, null, `${key} is missing a placeholder: ${translation}`);
+      assert.strictEqual(
+          tMatch.sort().toString(),
+          enMatch.sort().toString(),
+          `${key} is missing or has duplicate placeholders: ${translation}`
+      );
+    }
+    assert.strictEqual(translation.match(/[\n]/), null, `${key} contains a newline character ${translation}`);
+};
+
+const validate = function (json, name) {
+    assert.strictEqual(Object.keys(json).sort().toString(), enKeys, `${name}: Locale json keys do not match en.json`);
+    Object.entries(json).forEach(validateEntry);
+};
+
+let file = `// This file was automatically generated.  Do not modify.
+
+'use strict';
+
+goog.provide('Blockly.ScratchMsgs.allLocales');
+
+goog.require('Blockly.ScratchMsgs');
+
+`;
+
+let localeMap = {
+  'aa-dj': 'aa_DJ',
+  'es-419': 'es_419',
+  'pt-br': 'pt_BR',
+  'zh-cn': 'zh_CN',
+  'zh-tw': 'zh_TW'
+};
+
+const getLocaleData = async function (locale) {
+    let txLocale = localeMap[locale] || locale;
+    const data = await txPull(PROJECT, RESOURCE, txLocale, MODE);
+    return {
+        locale: locale,
+        translations: data
+    };
+};
+
+Promise.all(Object.keys(locales).map(getLocaleData)).then(function (values) {
+  values.forEach(function (translation) {
+    validate(translation.translations, translation.locale);
+    file += '\n';
+    file += `Blockly.ScratchMsgs.locales["${translation.locale}"] =\n`;
+    file += JSON.stringify(translation.translations, null, 4);
+    file += ';\n';
+  });
+  file += '// End of combined translations\n';
+  // write combined file
+  fs.writeFileSync(`${PATH_OUTPUT}/scratch_msgs.js`, file);
+}).catch((err) => {
+  console.error(err);
+  process.exit(1);
+});

package/i18n/test_scratch_msgs.js

@@ -0,0 +1,87 @@
+const es = require('event-stream');
+const fs = require('fs');
+const path = require('path');
+const assert = require('assert');
+
+// current locale and keys for the locale
+let locale = '';
+let keys = [];
+
+// current English keys
+let en = fs.readFileSync(path.resolve(__dirname, '../msg/json/en.json'));
+en = JSON.parse(en);
+const enKeys = Object.keys(en);
+
+// File paths
+const PATH_INPUT = path.resolve(__dirname, '../msg/scratch_msgs.js');
+
+// Match lines of the scratch_msgs file
+// Blockly.ScratchMsgs.locales indicates the start of a new locale
+// ": " marks a "key": "value" pair
+// Also match the end of the generated file so the last set of keys can be checked
+const match = function (str) {
+    if (str.indexOf('Blockly.ScratchMsgs.locales') !== 0) return true;
+    if (str.indexOf('": "') !== 0) return true;
+    if (str.indexOf('End of combined translations') !== 0) return true;
+    return false;
+}
+
+// Extract key and value from message definition, or locale when it changes
+const extract = function (str) {
+    let m = str.match(/locales\["(.+)"\]/);
+    if (m) {
+      // locale changed
+      return m[1];
+    }
+    m = str.match(/"(.*)": "(.*)",?$/);
+    if (m) {
+      return {
+        key: m[1],
+        value: m[2]
+      }
+    }
+    // return a string for the end of the file so that validate will check the last set of keys
+    m = str.match(/^\/\/ End of combined translations$/);
+    if (m) return 'last';
+    return null;
+};
+
+const validateKeys = function () {
+    // ignore empty keys first time through
+    if (keys.length === 0) return;
+    assert.strictEqual(keys.length, Object.keys(en).length, `scratch_msgs-${locale}: number of keys doesn't match`);
+    keys.map(item => assert(enKeys.includes(item), `scratch_msgs-${locale}: has key ${item} not in en`));
+    enKeys.map(item => assert(keys.includes(item), `scratch_msgs-${locale}: is missing key ${item}`));
+}
+
+// Stream input and push each match to the storage object
+const stream = fs.createReadStream(PATH_INPUT);
+stream
+    .pipe(es.split('\n'))
+    .pipe(es.mapSync(function (str) {
+        if (!match(str)) return;
+        const result = extract(str);
+        if (!result) return;
+        if (typeof result === 'string') {
+          // locale changed or end of file, validate the current collection of keys
+          try {
+              validateKeys();
+          }
+          catch (err) {
+            console.error('Key validation FAILED: %O', err);
+            process.exit(1);
+          }
+          // change locale, and reset keys array
+          locale = result;
+          keys = [];
+        } else {
+          keys.push(result.key);
+        }
+    }))
+    .pipe(es.wait(function (err) {
+        if (err) {
+          console.err(err);
+          process.exit(1);
+        }
+        process.exit(0)
+    }));

package/i18n/tests.py

@@ -0,0 +1,47 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# Tests of i18n scripts.
+#
+# Copyright 2013 Google Inc.
+# https://developers.google.com/blockly/
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import common
+import re
+import unittest
+
+class TestSequenceFunctions(unittest.TestCase):
+  def test_insert_breaks(self):
+    spaces = re.compile(r'\s+|\\n')
+    def contains_all_chars(orig, result):
+      return re.sub(spaces, '', orig) == re.sub(spaces, '', result)
+
+    sentences = [u'Quay Pegman qua bên trái hoặc bên phải 90 độ.',
+                 u'Foo bar baz this is english that is okay bye.',
+                 u'If there is a path in the specified direction, \nthen ' +
+                 u'do some actions.',
+                 u'If there is a path in the specified direction, then do ' +
+                 u'the first block of actions. Otherwise, do the second ' +
+                 u'block of actions.']
+    for sentence in sentences:
+      output = common.insert_breaks(sentence, 30, 50)
+      self.assertTrue(contains_all_chars(sentence, output),
+                   u'Mismatch between:\n{0}\n{1}'.format(
+                       re.sub(spaces, '', sentence),
+                       re.sub(spaces, '', output)))
+
+
+if __name__ == '__main__':
+    unittest.main()