jekyll-open-sdg-plugins 1.6.1 → 1.8.0.pre.beta1

data/lib/jekyll-open-sdg-plugins/fetch_remote_data.rb

@@ -1,188 +1,188 @@
-require "jekyll"
-require 'json'
-require 'deep_merge'
-require 'open-uri'
-require_relative "helpers"
-
-module JekyllOpenSdgPlugins
-  class FetchRemoteData < Jekyll::Generator
-    safe true
-    priority :highest
-
-    # Fix a Unix path in case we are on Windows.
-    def fix_path(path)
-      path_parts = path.split('/')
-      return path_parts.join(File::SEPARATOR)
-    end
-
-    # Our hardcoded list of pieces of the build that we expect.
-    def get_endpoints()
-      return {
-        'meta' => 'meta/all.json',
-        'headlines' => 'headline/all.json',
-        'schema' => 'meta/schema.json',
-        'reporting' => 'stats/reporting.json',
-        'disaggregation' => 'stats/disaggregation.json',
-        'translations' => 'translations/translations.json',
-        'zip' => 'zip/all_indicators.json',
-        'indicator_downloads' => 'downloads/indicator-downloads.json',
-        'data_packages' => 'data-packages/all.json',
-      }
-    end
-
-    # Get a build from a local folder on disk or a remote URL on the Internet.
-    def fetch_build(path)
-
-      is_remote = opensdg_is_path_remote(path)
-      build = {}
-      get_endpoints().each do |key, value|
-        endpoint = is_remote ? path + '/' + value : File.join(path, fix_path(value))
-
-        begin
-          json_file = is_remote ? open(endpoint) : File.open(endpoint)
-          build[key] = JSON.load(json_file)
-        rescue StandardError => e
-          # For backwards compatibility, forego the exception in some cases.
-          abort_build = true
-          if ['translations', 'indicator_downloads', 'disaggregation', 'data_packages'].include? key
-            abort_build = false
-          elsif endpoint.include? '/untranslated/'
-            abort_build = false
-          end
-          if abort_build
-            puts e.message
-            abort 'Unable to read data from: ' + endpoint
-          end
-        end
-      end
-
-      return build
-    end
-
-    # Predict (before data has been fetched) whether the site is using
-    # translated builds or not.
-    def site_uses_translated_builds(path)
-
-      is_remote = opensdg_is_path_remote(path)
-      endpoints = get_endpoints()
-      # For a quick test, we just use 'meta'.
-      meta = endpoints['meta']
-      endpoint = is_remote ? path + '/' + meta : File.join(path, fix_path(meta))
-
-      begin
-        json_file = is_remote ? open(endpoint) : File.open(endpoint)
-      rescue StandardError => e
-        # If we didn't find an untranslated 'meta', we assume translated builds.
-        return true
-      end
-
-      # Other wise assume untranslated builds.
-      return false
-    end
-
-    def generate(site)
-
-      # For below, make sure there is at least an empty hash at
-      # site.data.translations.
-      if !site.data.has_key?('translations')
-        site.data['translations'] = {}
-      end
-
-      remote = site.config['remote_data_prefix']
-      local = site.config['local_data_folder']
-
-      if !remote && !local
-        abort 'Site config must include "remote_data_prefix".'
-      end
-
-      build_location = remote ? remote : local
-      is_remote = opensdg_is_path_remote(build_location)
-
-      build_location = is_remote ? build_location : File.join(Dir.pwd, build_location)
-      translated_builds = site_uses_translated_builds(build_location)
-
-      if translated_builds
-        # For translated builds, we get a build for each language, and
-        # place them in "subfolders" (so to speak) of site.data.
-        subfolders = site.config['languages'].clone
-        subfolders.append('untranslated')
-        subfolders.each do |language|
-          data_target = site.data[language]
-          translated_build = is_remote ? build_location + '/' + language : File.join(build_location, language)
-          data_source = fetch_build(translated_build)
-          if !data_source.empty?
-            if data_target
-              data_target.deep_merge(data_source)
-            else
-              site.data[language] = data_source
-            end
-          end
-        end
-        # We move the language-specific translations to the
-        # site.data.translations location, where all translations are kept.
-        site.config['languages'].each do |language|
-          translation_target = site.data['translations'][language]
-          translation_source = site.data[language]['translations']
-          if translation_target
-            translation_target.deep_merge(translation_source)
-          else
-            site.data['translations'][language] = translation_source
-          end
-        end
-        # And there are some parts of the build that don't need to be translated
-        # and should be moved to the top level.
-        first_language = site.config['languages'][0]
-        site.data['reporting'] = site.data[first_language]['reporting']
-        site.data['schema'] = site.data[first_language]['schema']
-        site.data['zip'] = site.data[first_language]['zip']
-      else
-        # For untranslated builds, we download one build only, and place it
-        # in the "root" (so to speak) of site.data. Nothing else is needed.
-        target = site.data
-        source = fetch_build(build_location)
-        if !source.empty?
-          target.deep_merge(source)
-        end
-      end
-
-      # Finally support the deprecated 'remote_translations' option.
-      # This is deprecated because translations should now be in the
-      # data repository, where they will be fetched in fetch_build().
-      if site.config['remote_translations']
-        key = 'translations'
-        target = site.data[key]
-        site.config['remote_translations'].each do |endpoint|
-          begin
-            source = JSON.load(open(endpoint))
-            if target
-              target.deep_merge(source)
-            else
-              site.data[key] = source
-            end
-          rescue StandardError => e
-            puts e.message
-            abort 'Unable to fetch remote translation from: ' + endpoint
-          end
-        end
-      end
-    end
-  end
-
-  # This makes sure that the contents of the "local_data_folder" get copied
-  # into the Jekyll build, so that they can be served from the website.
-  Jekyll::Hooks.register :site, :post_write do |site|
-    if site.config['local_data_folder']
-      source = File.join(Dir.pwd, site.config['local_data_folder'], '.')
-      destination = site.config['destination']
-      FileUtils.cp_r(source, destination)
-    # Do the same in the case that "remote_data_prefix" is being used for a local
-    # data folder (since "local_data_folder" is deprecated and undocumented).
-    elsif site.config['remote_data_prefix']
-      if !opensdg_is_path_remote(site.config['remote_data_prefix'])
-        source = File.join(Dir.pwd, site.config['remote_data_prefix'], '.')
-        destination = site.config['destination']
-        FileUtils.cp_r(source, destination)
-      end
-    end
-  end
-end
+require "jekyll"
+require 'json'
+require 'deep_merge'
+require 'open-uri'
+require_relative "helpers"
+
+module JekyllOpenSdgPlugins
+  class FetchRemoteData < Jekyll::Generator
+    safe true
+    priority :highest
+
+    # Fix a Unix path in case we are on Windows.
+    def fix_path(path)
+      path_parts = path.split('/')
+      return path_parts.join(File::SEPARATOR)
+    end
+
+    # Our hardcoded list of pieces of the build that we expect.
+    def get_endpoints()
+      return {
+        'meta' => 'meta/all.json',
+        'headlines' => 'headline/all.json',
+        'schema' => 'meta/schema.json',
+        'reporting' => 'stats/reporting.json',
+        'disaggregation' => 'stats/disaggregation.json',
+        'translations' => 'translations/translations.json',
+        'zip' => 'zip/all_indicators.json',
+        'indicator_downloads' => 'downloads/indicator-downloads.json',
+        'data_packages' => 'data-packages/all.json',
+      }
+    end
+
+    # Get a build from a local folder on disk or a remote URL on the Internet.
+    def fetch_build(path)
+
+      is_remote = opensdg_is_path_remote(path)
+      build = {}
+      get_endpoints().each do |key, value|
+        endpoint = is_remote ? path + '/' + value : File.join(path, fix_path(value))
+
+        begin
+          json_file = is_remote ? open(endpoint) : File.open(endpoint)
+          build[key] = JSON.load(json_file)
+        rescue StandardError => e
+          # For backwards compatibility, forego the exception in some cases.
+          abort_build = true
+          if ['translations', 'indicator_downloads', 'disaggregation', 'data_packages'].include? key
+            abort_build = false
+          elsif endpoint.include? '/untranslated/'
+            abort_build = false
+          end
+          if abort_build
+            puts e.message
+            abort 'Unable to read data from: ' + endpoint
+          end
+        end
+      end
+
+      return build
+    end
+
+    # Predict (before data has been fetched) whether the site is using
+    # translated builds or not.
+    def site_uses_translated_builds(path)
+
+      is_remote = opensdg_is_path_remote(path)
+      endpoints = get_endpoints()
+      # For a quick test, we just use 'meta'.
+      meta = endpoints['meta']
+      endpoint = is_remote ? path + '/' + meta : File.join(path, fix_path(meta))
+
+      begin
+        json_file = is_remote ? open(endpoint) : File.open(endpoint)
+      rescue StandardError => e
+        # If we didn't find an untranslated 'meta', we assume translated builds.
+        return true
+      end
+
+      # Other wise assume untranslated builds.
+      return false
+    end
+
+    def generate(site)
+
+      # For below, make sure there is at least an empty hash at
+      # site.data.translations.
+      if !site.data.has_key?('translations')
+        site.data['translations'] = {}
+      end
+
+      remote = site.config['remote_data_prefix']
+      local = site.config['local_data_folder']
+
+      if !remote && !local
+        abort 'Site config must include "remote_data_prefix".'
+      end
+
+      build_location = remote ? remote : local
+      is_remote = opensdg_is_path_remote(build_location)
+
+      build_location = is_remote ? build_location : File.join(Dir.pwd, build_location)
+      translated_builds = site_uses_translated_builds(build_location)
+
+      if translated_builds
+        # For translated builds, we get a build for each language, and
+        # place them in "subfolders" (so to speak) of site.data.
+        subfolders = site.config['languages'].clone
+        subfolders.append('untranslated')
+        subfolders.each do |language|
+          data_target = site.data[language]
+          translated_build = is_remote ? build_location + '/' + language : File.join(build_location, language)
+          data_source = fetch_build(translated_build)
+          if !data_source.empty?
+            if data_target
+              data_target.deep_merge(data_source)
+            else
+              site.data[language] = data_source
+            end
+          end
+        end
+        # We move the language-specific translations to the
+        # site.data.translations location, where all translations are kept.
+        site.config['languages'].each do |language|
+          translation_target = site.data['translations'][language]
+          translation_source = site.data[language]['translations']
+          if translation_target
+            translation_target.deep_merge(translation_source)
+          else
+            site.data['translations'][language] = translation_source
+          end
+        end
+        # And there are some parts of the build that don't need to be translated
+        # and should be moved to the top level.
+        first_language = site.config['languages'][0]
+        site.data['reporting'] = site.data[first_language]['reporting']
+        site.data['schema'] = site.data[first_language]['schema']
+        site.data['zip'] = site.data[first_language]['zip']
+      else
+        # For untranslated builds, we download one build only, and place it
+        # in the "root" (so to speak) of site.data. Nothing else is needed.
+        target = site.data
+        source = fetch_build(build_location)
+        if !source.empty?
+          target.deep_merge(source)
+        end
+      end
+
+      # Finally support the deprecated 'remote_translations' option.
+      # This is deprecated because translations should now be in the
+      # data repository, where they will be fetched in fetch_build().
+      if site.config['remote_translations']
+        key = 'translations'
+        target = site.data[key]
+        site.config['remote_translations'].each do |endpoint|
+          begin
+            source = JSON.load(open(endpoint))
+            if target
+              target.deep_merge(source)
+            else
+              site.data[key] = source
+            end
+          rescue StandardError => e
+            puts e.message
+            abort 'Unable to fetch remote translation from: ' + endpoint
+          end
+        end
+      end
+    end
+  end
+
+  # This makes sure that the contents of the "local_data_folder" get copied
+  # into the Jekyll build, so that they can be served from the website.
+  Jekyll::Hooks.register :site, :post_write do |site|
+    if site.config['local_data_folder']
+      source = File.join(Dir.pwd, site.config['local_data_folder'], '.')
+      destination = site.config['destination']
+      FileUtils.cp_r(source, destination)
+    # Do the same in the case that "remote_data_prefix" is being used for a local
+    # data folder (since "local_data_folder" is deprecated and undocumented).
+    elsif site.config['remote_data_prefix']
+      if !opensdg_is_path_remote(site.config['remote_data_prefix'])
+        source = File.join(Dir.pwd, site.config['remote_data_prefix'], '.')
+        destination = site.config['destination']
+        FileUtils.cp_r(source, destination)
+      end
+    end
+  end
+end

data/lib/jekyll-open-sdg-plugins/helpers.rb

@@ -1,132 +1,132 @@
-# Simple collection of helper functions for use in these plugins.
-
-require "jekyll"
-
-# Takes a translation key and returns a translated string according to the
-# language of the current page. Or if none is found, returns the original
-# key.
-def opensdg_translate_key(key, translations, language)
-
-  # Safety code - abort now if key is nil.
-  if key.nil?
-    return ""
-  end
-
-  # Also make sure it is a string, and other just return it.
-  if not key.is_a? String
-    return key
-  end
-
-  # More safety code - abort now if key is empty.
-  if key.empty?
-    return ""
-  end
-
-  # Keep track of the last thing we drilled to.
-  drilled = translations[language]
-
-  # Keep track of how many levels we have drilled.
-  levels_drilled = 0
-  levels = key.split('.')
-
-  # Loop through each level.
-  levels.each do |level|
-
-    # If we have drilled down to a scalar value too soon, abort.
-    break if drilled.class != Hash
-
-    if drilled.has_key? level
-      # If we find something, continue drilling.
-      drilled = drilled[level]
-      levels_drilled += 1
-    end
-
-  end
-
-  # If we didn't drill the right number of levels, return the
-  # original string.
-  if levels.length != levels_drilled
-    return key
-  end
-
-  # Otherwise we must have drilled all they way.
-  return drilled
-end
-
-# Takes a site object and decides whether it is using translated builds.
-def opensdg_translated_builds(site)
-  # Assume the site is using translated builds.
-  translated_builds = true
-  site.config['languages'].each do |language|
-    # If any languages don't have a key in site.data, the site is not using
-    # translated builds.
-    if !site.data.has_key? language
-      translated_builds = false
-    end
-  end
-  return translated_builds
-end
-
-# Print a notice during compilation.
-def opensdg_notice(message)
-  Jekyll.logger.warn message.yellow
-end
-
-# Get the public language codes for a site, keyed by the actual language codes.
-def opensdg_languages_public(site)
-  languages_public = site.config['languages_public']
-
-  # The current structure of the setting is an array of hashes, each containing
-  # keys for "language" and "language_public".
-  if languages_public.is_a?(Array)
-    converted_languages_public = Hash.new
-    languages_public.each do |language_public|
-      language_code = language_public['language']
-      language_code_public = language_public['language_public']
-      converted_languages_public[language_code] = language_code_public
-    end
-    return converted_languages_public
-  end
-
-  # Fallback to exactly what was retrieved from site.confg['languages_public'],
-  # since the deprecated structure is exactly what this function wants.
-  return languages_public
-end
-
-# Print notices about a validation error.
-def opensdg_validation_error(error)
-  if error['type'] == 'required'
-    missing = []
-    error['schema']['required'].each do |required_property|
-      unless error['data'].has_key?(required_property)
-        message = 'Missing configuration setting: ' + required_property
-        if error['schema'].has_key?('title')
-          message += ' (' + error['schema']['title'] + ')'
-        end
-        opensdg_notice(message)
-      end
-    end
-  else
-    message = 'Validation error of type: ' + error['type']
-    if error['schema'] && error['schema'].has_key?('title')
-      message += ' (' + error['schema']['title'] + ')'
-    end
-    opensdg_notice(message)
-    if error['schema']
-      opensdg_notice('Expected schema:')
-      puts error['schema'].inspect
-    end
-    if error['data']
-      opensdg_notice('Actual data:')
-      puts error['data'].inspect
-    end
-  end
-end
-
-# Is this path a remote path?
-def opensdg_is_path_remote(path)
-  if path.nil?
-    return false
-  end
-  return path.start_with?('http')
-end
+# Simple collection of helper functions for use in these plugins.
+
+require "jekyll"
+
+# Takes a translation key and returns a translated string according to the
+# language of the current page. Or if none is found, returns the original
+# key.
+def opensdg_translate_key(key, translations, language)
+
+  # Safety code - abort now if key is nil.
+  if key.nil?
+    return ""
+  end
+
+  # Also make sure it is a string, and other just return it.
+  if not key.is_a? String
+    return key
+  end
+
+  # More safety code - abort now if key is empty.
+  if key.empty?
+    return ""
+  end
+
+  # Keep track of the last thing we drilled to.
+  drilled = translations[language]
+
+  # Keep track of how many levels we have drilled.
+  levels_drilled = 0
+  levels = key.split('.')
+
+  # Loop through each level.
+  levels.each do |level|
+
+    # If we have drilled down to a scalar value too soon, abort.
+    break if drilled.class != Hash
+
+    if drilled.has_key? level
+      # If we find something, continue drilling.
+      drilled = drilled[level]
+      levels_drilled += 1
+    end
+
+  end
+
+  # If we didn't drill the right number of levels, return the
+  # original string.
+  if levels.length != levels_drilled
+    return key
+  end
+
+  # Otherwise we must have drilled all they way.
+  return drilled
+end
+
+# Takes a site object and decides whether it is using translated builds.
+def opensdg_translated_builds(site)
+  # Assume the site is using translated builds.
+  translated_builds = true
+  site.config['languages'].each do |language|
+    # If any languages don't have a key in site.data, the site is not using
+    # translated builds.
+    if !site.data.has_key? language
+      translated_builds = false
+    end
+  end
+  return translated_builds
+end
+
+# Print a notice during compilation.
+def opensdg_notice(message)
+  Jekyll.logger.warn message.yellow
+end
+
+# Get the public language codes for a site, keyed by the actual language codes.
+def opensdg_languages_public(site)
+  languages_public = site.config['languages_public']
+
+  # The current structure of the setting is an array of hashes, each containing
+  # keys for "language" and "language_public".
+  if languages_public.is_a?(Array)
+    converted_languages_public = Hash.new
+    languages_public.each do |language_public|
+      language_code = language_public['language']
+      language_code_public = language_public['language_public']
+      converted_languages_public[language_code] = language_code_public
+    end
+    return converted_languages_public
+  end
+
+  # Fallback to exactly what was retrieved from site.confg['languages_public'],
+  # since the deprecated structure is exactly what this function wants.
+  return languages_public
+end
+
+# Print notices about a validation error.
+def opensdg_validation_error(error)
+  if error['type'] == 'required'
+    missing = []
+    error['schema']['required'].each do |required_property|
+      unless error['data'].has_key?(required_property)
+        message = 'Missing configuration setting: ' + required_property
+        if error['schema'].has_key?('title')
+          message += ' (' + error['schema']['title'] + ')'
+        end
+        opensdg_notice(message)
+      end
+    end
+  else
+    message = 'Validation error of type: ' + error['type']
+    if error['schema'] && error['schema'].has_key?('title')
+      message += ' (' + error['schema']['title'] + ')'
+    end
+    opensdg_notice(message)
+    if error['schema']
+      opensdg_notice('Expected schema:')
+      puts error['schema'].inspect
+    end
+    if error['data']
+      opensdg_notice('Actual data:')
+      puts error['data'].inspect
+    end
+  end
+end
+
+# Is this path a remote path?
+def opensdg_is_path_remote(path)
+  if path.nil?
+    return false
+  end
+  return path.start_with?('http')
+end