wca_i18n 0.2.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 52de91ef7d972252dd256379fe3d091743f72d4e
+  data.tar.gz: b814ae9af19eba3e04e1a48feb02954637a2fc61
+SHA512:
+  metadata.gz: 8d15b262f498624b3a92a272dee1723e84b69081ef087d5c1d0550f11e58a00dee380103e1d8d22b3882928d28b685c933bcf59e50b375873e56e4bfb7488900
+  data.tar.gz: 72ef348d65fd55382f5f7ad7cd42162fc7f03260a90b52e257d48e7aa9964f91d79c9c9ec505e5c402f5a30a9874b8ee99a9f2355676d8f33c0dae0a3a90bb32

data/.editorconfig

@@ -0,0 +1,9 @@
+root = true
+
+[*]
+end_of_line = lf
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true
+indent_style = space
+indent_size = 2

data/.gitignore

@@ -0,0 +1,13 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status
+
+.byebug_history

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.4.3
+before_install: gem install bundler -v 1.16.0

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in wca_i18n.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,37 @@
+PATH
+  remote: .
+  specs:
+    wca_i18n (0.2.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    byebug (9.1.0)
+    diff-lcs (1.3)
+    rake (10.5.0)
+    rspec (3.7.0)
+      rspec-core (~> 3.7.0)
+      rspec-expectations (~> 3.7.0)
+      rspec-mocks (~> 3.7.0)
+    rspec-core (3.7.1)
+      rspec-support (~> 3.7.0)
+    rspec-expectations (3.7.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.7.0)
+    rspec-mocks (3.7.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.7.0)
+    rspec-support (3.7.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.16)
+  byebug (~> 9.0)
+  rake (~> 10.0)
+  rspec (~> 3.0)
+  wca_i18n!
+
+BUNDLED WITH
+   1.16.0

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2018 Jeremy Fleischman
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,52 @@
+# WcaI18n [![Build Status](https://travis-ci.org/thewca/wca-i18n.svg?branch=master)](https://travis-ci.org/thewca/wca-i18n)
+
+Use this Gem to diff Rails translations.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'wca_i18n'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install wca_i18n
+
+## Usage
+
+### Using the wca_i18n executable
+
+Use the included `wca_i18n` binary to compare translations to a base translation to see
+how out of date they are.
+
+```bash
+$ wca_i18n en.yml *.yml
+```
+
+### Using the wca_i18n library
+
+There are two parts to this library: `WcaI18n::YAMLWithComments` (a YAML parser
+that preserves comments) and `WcaI18n::Translation` (used to load and diff
+translation YAML files). Until we have better documentation, it's best to look
+at our [specs](https://github.com/thewca/wca-i18n/tree/master/spec) for how to
+them.
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `bin/rspec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/thewca/wca-i18n.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "wca_i18n"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/rspec

@@ -0,0 +1,21 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+#
+# This file was generated by Bundler.
+#
+# The application 'rspec' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+bundle_binstub = File.expand_path("../bundle", __FILE__)
+load(bundle_binstub) if File.file?(bundle_binstub)
+
+require "pathname"
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+require "rubygems"
+require "bundler/setup"
+
+load Gem.bin_path("rspec-core", "rspec")

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/exe/wca_i18n

@@ -0,0 +1,66 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+require 'wca_i18n'
+
+def run!
+  verbose = false
+  OptionParser.new do |opts|
+    opts.banner = "Usage: #{__FILE__} [options] <base_translation_file> <translation_files>"
+    opts.on("-v", "--verbose", "Run verbosely") do |v|
+      verbose = v
+    end
+  end.parse!
+
+  base_translation_file = ARGV.shift
+  throw "Please pass a base_translation_file" unless base_translation_file
+  throw "base_translation_file not found: #{base_translation_file}" unless File.exist?(base_translation_file)
+
+  translation_files = ARGV
+  throw "Please pass a list of translation_files" unless translation_files
+
+  base_translation = load_translation(base_translation_file)
+  translations = translation_files.map { |f| load_translation(f) }.reject { |t| t.locale == base_translation.locale }
+
+  compare_base_with_translations(base_translation, translations.sort_by!(&:locale), verbose)
+end
+
+def load_translation(filename)
+  locale = File.basename(filename, ".*")
+  WcaI18n::Translation.new(locale, File.read(filename))
+end
+
+def compare_base_with_translations(base_translation, translations, verbose)
+  translations.each do |translation|
+    diff = translation.compare_to(base_translation)
+    puts "#{translation.locale}:#{indent(format_diff(diff, verbose), 1)}"
+  end
+end
+
+def format_diff(diff, verbose=false)
+  types = [:missing, :outdated, :unused]
+
+  pretty_counts = types.map do |type|
+    contexts = diff[type]
+    pretty = "#{contexts.length} #{type.to_s}"
+    pretty += ":\n" + indent(format_contexts(contexts), 1) if verbose
+    pretty
+  end
+
+  if verbose
+    "\n" + pretty_counts.join("\n")
+  else
+    total = diff.values.map(&:length).sum
+    "#{total} total = #{pretty_counts.join(" + ")}"
+  end
+end
+
+def format_contexts(contexts)
+  contexts.map { |context| context.join(" > ") }.join("\n")
+end
+
+def indent(str, level)
+  str.split("\n").map { |line| "\t"*level + line }.join("\n")
+end
+
+run! if __FILE__ == $0

data/lib/wca_i18n/translation.rb

@@ -0,0 +1,103 @@
+require "json"
+require "digest"
+require "wca_i18n/yaml_with_comments"
+
+module WcaI18n
+  TranslatedLeaf = Struct.new(:translated, :original_hash)
+
+  class Translation
+    PLURALIZATION_KEYS = %w(zero one two few many other).freeze
+
+    attr_accessor :locale, :data
+
+    def initialize(locale, file_content)
+      self.locale = locale.to_s
+      self.data = commented_yaml_to_translated_yaml(YAMLWithComments.parse(file_content))
+    end
+
+    def compare_to(base)
+      return diff_recursive(base.data[base.locale], self.data[self.locale], [])
+    end
+
+    def self.hash_translation(value)
+      # If the key is a pluralization, we use all the subkeys to compute the hash
+      # Please see this wiki page explaining why we do this: https://github.com/thewca/worldcubeassociation.org/wikigTranslating-the-website#translations-status-internals
+      to_digest = pluralization?(value) ? JSON.generate(value) : value
+      original_str = Digest::SHA1.hexdigest(to_digest)[0..6]
+    end
+
+    private def commented_yaml_to_translated_yaml(commented_value)
+      if leaf?(commented_value.value)
+        original_hash = extract_original_hash_from_comment(commented_value.comment)
+        return TranslatedLeaf.new(commented_value.strip_comments, original_hash)
+      end
+
+      commented_value.value.map do |key, value|
+        [key, commented_yaml_to_translated_yaml(value)]
+      end.to_h
+    end
+
+    private def diff_recursive(base, translation, context)
+      diff = { missing: [], unused: [], outdated: [] }
+      base_leaf = base.is_a?(TranslatedLeaf)
+      translation_leaf = translation.is_a?(TranslatedLeaf)
+
+      if !base
+        diff[:unused] += get_all_recursive(translation, context)
+      elsif !translation
+        diff[:missing] += get_all_recursive(base, context)
+      elsif base_leaf && translation_leaf
+        if translation.translated.nil?
+          diff[:missing] << context
+        elsif self.class.hash_translation(base.translated) != translation.original_hash
+          diff[:outdated] << context
+        end
+      elsif base_leaf && !translation_leaf
+        diff[:missing] << context
+        diff[:unused] += get_all_recursive(translation, context)
+      elsif !base_leaf && translation_leaf
+        diff[:missing] += get_all_recursive(base, context)
+        diff[:unused] << context
+      else
+        (base.keys | translation.keys).each do |key|
+          merge_diffs!(diff, diff_recursive(base[key], translation[key], [*context, key]))
+        end
+      end
+
+      diff
+    end
+
+    private def merge_diffs!(diff, other_diff)
+      other_diff.each do |key, value|
+        diff[key] += value
+      end
+      diff
+    end
+
+    private def get_all_recursive(node, context)
+      return [ context ] if node.is_a?(TranslatedLeaf)
+
+      node.map { |key, value| get_all_recursive(value, [*context, key]) }.flatten(1)
+    end
+
+    private def extract_original_hash_from_comment(comment)
+      hashes = comment.scan(/original_hash:\s*(.+)/).flatten(1)
+      if hashes.size == 0
+        nil
+      elsif hashes.size == 1
+        hashes.first
+      elsif hashes.size > 1
+        throw "Too many #{HASHTAG} occurrences in: #{comment}"
+      end
+    end
+
+    private def leaf?(node)
+      # If the node is a pluralization it's also a leaf!
+      node.nil? || node.is_a?(String) || self.class.pluralization?(node)
+    end
+
+    def self.pluralization?(node)
+      node.is_a?(Hash) && (node.keys & PLURALIZATION_KEYS).any?
+    end
+  end
+end

data/lib/wca_i18n/version.rb

@@ -0,0 +1,3 @@
+module WcaI18n
+  VERSION = "0.2.0"
+end

data/lib/wca_i18n/yaml_with_comments.rb

@@ -0,0 +1,68 @@
+require "yaml"
+
+module WcaI18n
+  class YAMLWithComments < Struct.new(:comment, :value)
+    def self.parse(text)
+      _decorate_with_comments(YAML.safe_load(text), [], text)[1]
+    end
+
+    def strip_comments
+      stripped_value = self.value
+
+      if stripped_value.kind_of?(Hash)
+        stripped_value = stripped_value.map do |key, value|
+          [key, value.strip_comments]
+        end.to_h
+      end
+
+      stripped_value
+    end
+
+    def self._decorate_with_comments(node, context, text)
+      if node.kind_of?(Hash)
+        node = node.map do |key, value|
+          text, decorated = _decorate_with_comments(value, [*context, key], text)
+          [key, decorated]
+        end.to_h
+      end
+
+      trimmed_text, comment = _extract_comment(text, context)
+      [trimmed_text, YAMLWithComments.new(comment, node)]
+    end
+
+    def self._extract_comment(text, context)
+      return ["", ""] if context.empty?
+
+      # Match any character, including newline.
+      some_chars = /[\s\S]*?/
+
+      comment_lines_group = /((?:^\s*#.*\n)*)/
+
+      match_parent = ""
+      context[0...-1].each do |key|
+        match_parent = "#{match_parent}#{some_chars}#{build_key_matcher(key)}"
+      end
+      regexp = /(#{match_parent}#{some_chars})#{comment_lines_group}\s*#{build_key_matcher(context[-1])}/
+
+      comment = nil
+      text = text.sub(regexp) do
+        # Group 1 is everything before the comment and the key.
+        before = $1
+        # Group 2 contains the comments matched before the key.
+        comments = $2
+        comment = comments.split('#').map(&:strip).reject(&:empty?).join("\n")
+
+        # We return the beginning without the key, so that the current hash + key
+        # are removed from the text, but the parents and the value stay in.
+        before
+      end
+      throw "Could not find key: #{context} in given yaml text" unless comment
+
+      [text, comment]
+    end
+
+    def self.build_key_matcher(key)
+      /['\"]?#{key}['\"]?:/
+    end
+  end
+end

data/lib/wca_i18n.rb

@@ -0,0 +1,2 @@
+require "wca_i18n/version"
+require "wca_i18n/translation"

data/wca_i18n.gemspec

@@ -0,0 +1,27 @@
+
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "wca_i18n/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "wca_i18n"
+  spec.version       = WcaI18n::VERSION
+  spec.authors       = ["Jeremy Fleischman"]
+  spec.email         = ["software@worldcubeassociation.org"]
+
+  spec.summary       = %q{Track how up to date Rails translations are.}
+  spec.homepage      = "https://github.com/thewca/wca_i18n"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.16"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "byebug", "~> 9.0"
+end

metadata

@@ -0,0 +1,119 @@
+--- !ruby/object:Gem::Specification
+name: wca_i18n
+version: !ruby/object:Gem::Version
+  version: 0.2.0
+platform: ruby
+authors:
+- Jeremy Fleischman
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2018-01-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '9.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '9.0'
+description: 
+email:
+- software@worldcubeassociation.org
+executables:
+- wca_i18n
+extensions: []
+extra_rdoc_files: []
+files:
+- ".editorconfig"
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/rspec
+- bin/setup
+- exe/wca_i18n
+- lib/wca_i18n.rb
+- lib/wca_i18n/translation.rb
+- lib/wca_i18n/version.rb
+- lib/wca_i18n/yaml_with_comments.rb
+- wca_i18n.gemspec
+homepage: https://github.com/thewca/wca_i18n
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.14
+signing_key: 
+specification_version: 4
+summary: Track how up to date Rails translations are.
+test_files: []