cchardet 0.2.0

data/ext/uchardet/build-mac/uchardet.xcodeproj/project.xcworkspace/contents.xcworkspacedata

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Workspace
+   version = "1.0">
+   <FileRef
+      location = "self:uchardet.xcodeproj">
+   </FileRef>
+</Workspace>

data/ext/uchardet/build-mac/uchardet.xcodeproj/project.xcworkspace/xcshareddata/uchardet.xccheckout

@@ -0,0 +1,41 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>IDESourceControlProjectFavoriteDictionaryKey</key>
+	<false/>
+	<key>IDESourceControlProjectIdentifier</key>
+	<string>A3B241B4-458F-47BE-A7FB-AABC82F7C332</string>
+	<key>IDESourceControlProjectName</key>
+	<string>uchardet</string>
+	<key>IDESourceControlProjectOriginsDictionary</key>
+	<dict>
+		<key>7DF9952B76987F54D66D74171482E11FAF33FDCB</key>
+		<string>https://www.freedesktop.org/wiki/Software/uchardet/</string>
+	</dict>
+	<key>IDESourceControlProjectPath</key>
+	<string>build-mac/uchardet.xcodeproj</string>
+	<key>IDESourceControlProjectRelativeInstallPathDictionary</key>
+	<dict>
+		<key>7DF9952B76987F54D66D74171482E11FAF33FDCB</key>
+		<string>../../..</string>
+	</dict>
+	<key>IDESourceControlProjectURL</key>
+	<string>https://www.freedesktop.org/wiki/Software/uchardet/</string>
+	<key>IDESourceControlProjectVersion</key>
+	<integer>111</integer>
+	<key>IDESourceControlProjectWCCIdentifier</key>
+	<string>7DF9952B76987F54D66D74171482E11FAF33FDCB</string>
+	<key>IDESourceControlProjectWCConfigurations</key>
+	<array>
+		<dict>
+			<key>IDESourceControlRepositoryExtensionIdentifierKey</key>
+			<string>public.vcs.git</string>
+			<key>IDESourceControlWCCIdentifierKey</key>
+			<string>7DF9952B76987F54D66D74171482E11FAF33FDCB</string>
+			<key>IDESourceControlWCCName</key>
+			<string>uchardet</string>
+		</dict>
+	</array>
+</dict>
+</plist>

data/ext/uchardet/build-mac/uchardet.xcodeproj/xcshareddata/xcschemes/uchardet-ios.xcscheme

@@ -0,0 +1,77 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Scheme
+   LastUpgradeVersion = "0600"
+   version = "1.3">
+   <BuildAction
+      parallelizeBuildables = "YES"
+      buildImplicitDependencies = "YES">
+      <BuildActionEntries>
+         <BuildActionEntry
+            buildForTesting = "YES"
+            buildForRunning = "YES"
+            buildForProfiling = "YES"
+            buildForArchiving = "YES"
+            buildForAnalyzing = "YES">
+            <BuildableReference
+               BuildableIdentifier = "primary"
+               BlueprintIdentifier = "BD49961119FC04E8000945BC"
+               BuildableName = "libuchardet-ios.a"
+               BlueprintName = "uchardet-ios"
+               ReferencedContainer = "container:uchardet.xcodeproj">
+            </BuildableReference>
+         </BuildActionEntry>
+      </BuildActionEntries>
+   </BuildAction>
+   <TestAction
+      selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
+      selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
+      shouldUseLaunchSchemeArgsEnv = "YES"
+      buildConfiguration = "Debug">
+      <Testables>
+      </Testables>
+   </TestAction>
+   <LaunchAction
+      selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
+      selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
+      launchStyle = "0"
+      useCustomWorkingDirectory = "NO"
+      buildConfiguration = "Debug"
+      ignoresPersistentStateOnLaunch = "NO"
+      debugDocumentVersioning = "YES"
+      allowLocationSimulation = "YES">
+      <MacroExpansion>
+         <BuildableReference
+            BuildableIdentifier = "primary"
+            BlueprintIdentifier = "BD49961119FC04E8000945BC"
+            BuildableName = "libuchardet-ios.a"
+            BlueprintName = "uchardet-ios"
+            ReferencedContainer = "container:uchardet.xcodeproj">
+         </BuildableReference>
+      </MacroExpansion>
+      <AdditionalOptions>
+      </AdditionalOptions>
+   </LaunchAction>
+   <ProfileAction
+      shouldUseLaunchSchemeArgsEnv = "YES"
+      savedToolIdentifier = ""
+      useCustomWorkingDirectory = "NO"
+      buildConfiguration = "Release"
+      debugDocumentVersioning = "YES">
+      <MacroExpansion>
+         <BuildableReference
+            BuildableIdentifier = "primary"
+            BlueprintIdentifier = "BD49961119FC04E8000945BC"
+            BuildableName = "libuchardet-ios.a"
+            BlueprintName = "uchardet-ios"
+            ReferencedContainer = "container:uchardet.xcodeproj">
+         </BuildableReference>
+      </MacroExpansion>
+   </ProfileAction>
+   <AnalyzeAction
+      buildConfiguration = "Debug">
+   </AnalyzeAction>
+   <ArchiveAction
+      buildConfiguration = "Release"
+      revealArchiveInOrganizer = "YES">
+   </ArchiveAction>
+</Scheme>

data/ext/uchardet/build-mac/uchardet.xcodeproj/xcshareddata/xcschemes/uchardet.xcscheme

@@ -0,0 +1,77 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Scheme
+   LastUpgradeVersion = "0600"
+   version = "1.3">
+   <BuildAction
+      parallelizeBuildables = "YES"
+      buildImplicitDependencies = "YES">
+      <BuildActionEntries>
+         <BuildActionEntry
+            buildForTesting = "YES"
+            buildForRunning = "YES"
+            buildForProfiling = "YES"
+            buildForArchiving = "YES"
+            buildForAnalyzing = "YES">
+            <BuildableReference
+               BuildableIdentifier = "primary"
+               BlueprintIdentifier = "BD49959B19F84840000945BC"
+               BuildableName = "libuchardet.a"
+               BlueprintName = "uchardet"
+               ReferencedContainer = "container:uchardet.xcodeproj">
+            </BuildableReference>
+         </BuildActionEntry>
+      </BuildActionEntries>
+   </BuildAction>
+   <TestAction
+      selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
+      selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
+      shouldUseLaunchSchemeArgsEnv = "YES"
+      buildConfiguration = "Debug">
+      <Testables>
+      </Testables>
+   </TestAction>
+   <LaunchAction
+      selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
+      selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
+      launchStyle = "0"
+      useCustomWorkingDirectory = "NO"
+      buildConfiguration = "Debug"
+      ignoresPersistentStateOnLaunch = "NO"
+      debugDocumentVersioning = "YES"
+      allowLocationSimulation = "YES">
+      <MacroExpansion>
+         <BuildableReference
+            BuildableIdentifier = "primary"
+            BlueprintIdentifier = "BD49959B19F84840000945BC"
+            BuildableName = "libuchardet.a"
+            BlueprintName = "uchardet"
+            ReferencedContainer = "container:uchardet.xcodeproj">
+         </BuildableReference>
+      </MacroExpansion>
+      <AdditionalOptions>
+      </AdditionalOptions>
+   </LaunchAction>
+   <ProfileAction
+      shouldUseLaunchSchemeArgsEnv = "YES"
+      savedToolIdentifier = ""
+      useCustomWorkingDirectory = "NO"
+      buildConfiguration = "Release"
+      debugDocumentVersioning = "YES">
+      <MacroExpansion>
+         <BuildableReference
+            BuildableIdentifier = "primary"
+            BlueprintIdentifier = "BD49959B19F84840000945BC"
+            BuildableName = "libuchardet.a"
+            BlueprintName = "uchardet"
+            ReferencedContainer = "container:uchardet.xcodeproj">
+         </BuildableReference>
+      </MacroExpansion>
+   </ProfileAction>
+   <AnalyzeAction
+      buildConfiguration = "Debug">
+   </AnalyzeAction>
+   <ArchiveAction
+      buildConfiguration = "Release"
+      revealArchiveInOrganizer = "YES">
+   </ArchiveAction>
+</Scheme>

data/ext/uchardet/doc/CMakeLists.txt

@@ -0,0 +1,6 @@
+install(
+	FILES
+		uchardet.1
+	DESTINATION
+		${CMAKE_INSTALL_MANDIR}/man1
+)

data/ext/uchardet/doc/README.maintainer

@@ -0,0 +1,59 @@
+# How to do a uchardet release #
+
+* Update UCHARDET_VERSION_MAJOR, UCHARDET_VERSION_MINOR and
+  UCHARDET_VERSION_REVISION as needed in CMakeLists.txt.
+
+* Update README.md.
+
+* Commit the version change with the message "Release: version X.Y.Z."
+
+* Tag your release commit with:
+
+    git tag -a vx.y.z
+
+  The tag message should be have the header "Version x.y.z released." followed
+  by a list of new features or important fixes. This tag message will be
+  considered as the release note, hence have to be carefully crafted.
+
+  Considering that the previous release was va.b.c, you can read the full list
+  of commits between a.b.c and x.y.z with:
+
+    git log va.b.c..
+
+  This should help you to build a proper release note.
+
+* Push the release and the tag:
+
+    git push
+    git push origin vx.y.z
+
+* Create a release tarball:
+
+    git archive --format=tar.xz --prefix=uchardet-x.y.z/ vx.y.z >uchardet-x.y.z.tar.xz
+
+Note: if you have not already set this up, you have to run first:
+
+    git config tar.tar.xz.command "xz -c"
+
+Cf. EXAMPLES section in `git help archive`.
+
+* Compute a SHA1 checksum:
+
+    sha1sum uchardet-x.y.x.tar.xz > uchardet-x.y.z.tar.xz.sha1
+
+* Upload to annarchy download server:
+
+    scp uchardet-x.y.x.tar.xz uchardet-x.y.z.tar.xz.sha1 annarchy.freedesktop.org:/srv/www.freedesktop.org/www/software/uchardet/releases/
+
+  The archive and its checksum file should now be available from:
+  https://www.freedesktop.org/software/uchardet/releases/
+
+* Make the git tag into a Gitlab release (not automatic).
+  It will be found at: https://gitlab.freedesktop.org/uchardet/uchardet/-/tags/vx.y.z
+  Just click the "Edit release notes" button, and copy paste the tag comment as "release notes".
+
+* Update the wiki page: https://www.freedesktop.org/wiki/Software/uchardet/
+  The release note link will be:
+  https://gitlab.freedesktop.org/uchardet/uchardet/-/releases/vx.y.z
+
+* Spread the good news!

data/ext/uchardet/doc/uchardet.1

@@ -0,0 +1,18 @@
+.TH UCHARDET "1" "July 2011" "uchardet " "User Commands"
+.SH NAME
+uchardet \- Universal Charset Detector
+.SH DESCRIPTION
+uchardet CLI is an encoding detector utility, which takes one or several files in unknown character encoding without any additional information, and attempts to determine the encoding of the texts. Returned encoding names are iconv-compatible.
+.SS "Usage:"
+.HP
+uchardet [\fBOptions\fR] [\fBFile\fR]...
+.HP
+\fB\-v\fR
+Print version and build information.
+.HP
+\fB\-h\fR
+Print help text.
+.HP
+.IP
+.PP
+uchardet Command Line Interface

data/ext/uchardet/script/BuildLangModel.py

@@ -0,0 +1,533 @@
+#!/bin/python3
+# -*- coding: utf-8 -*-
+
+# ##### BEGIN LICENSE BLOCK #####
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is Mozilla Universal charset detector code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 2001
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#          Jehan <jehan@girinstud.io>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ##### END LICENSE BLOCK #####
+
+# Third party modules.
+import unicodedata
+import subprocess
+import wikipedia
+import importlib
+import optparse
+import datetime
+import operator
+import requests
+import sys
+import re
+import os
+
+# Custom modules.
+import charsets.db
+from charsets.codepoints import *
+
+# Command line processing.
+usage = 'Usage: {} <LANG-CODE>\n' \
+        '\nEx: `{} fr`'.format(__file__, __file__)
+
+description = "Internal tool for uchardet to generate language data."
+cmdline = optparse.OptionParser(usage, description = description)
+cmdline.add_option('--max-page',
+                   help = 'Maximum number of Wikipedia pages to parse (useful for debugging).',
+                   action = 'store', type = 'int', dest = 'max_page', default = None)
+cmdline.add_option('--max-depth',
+                   help = 'Maximum depth when following links from start page (default: 2).',
+                   action = 'store', type = 'int',
+                   dest = 'max_depth', default = 2)
+(options, langs) = cmdline.parse_args()
+if len(langs) < 1:
+    print("Please select at least one language code.\n")
+    exit(1)
+if len(langs) > 1:
+    print("This script is meant to generate data for one language at a time.\n")
+    exit(1)
+lang = langs[0]
+
+# Load the language data.
+sys_path_backup = sys.path
+current_dir = os.path.dirname(os.path.realpath(__file__))
+sys.path = [current_dir + '/langs']
+
+try:
+    lang = importlib.import_module(lang.lower())
+except ImportError:
+    print('Unknown language code "{}": '
+          'file "langs/{}.py" does not exist.'.format(lang, lang.lower()))
+    exit(1)
+sys.path = sys_path_backup
+
+charsets = charsets.db.load(lang.charsets)
+
+if not hasattr(lang, 'start_pages') or lang.start_pages is None or \
+   lang.start_pages == []:
+    # Let's start with the main page, assuming it should have links
+    # to relevant pages. In locale wikipedia, this page is usually redirected
+    # to a relevant page.
+    print("Warning: no `start_pages` set for '{}'. Using ['Main_Page'].\n"
+          "         If you don't get good data, it is advised to set a "
+          "start_pages` variable yourself.".format(lang.code))
+    lang.start_pages = ['Main_Page']
+if not hasattr(lang, 'wikipedia_code') or lang.wikipedia_code is None:
+    lang.wikipedia_code = lang.code
+if not hasattr(lang, 'clean_wikipedia_content') or lang.clean_wikipedia_content is None:
+    lang.clean_wikipedia_content = None
+if hasattr(lang, 'case_mapping'):
+    lang.case_mapping = bool(lang.case_mapping)
+else:
+    lang.case_mapping = False
+if not hasattr(lang, 'custom_case_mapping'):
+    lang.custom_case_mapping = None
+if not hasattr(lang, 'alphabet') or lang.alphabet is None:
+    lang.alphabet = None
+
+def local_lowercase(text, lang):
+    lowercased = ''
+    for l in text:
+        if lang.custom_case_mapping is not None and \
+           l in lang.custom_case_mapping:
+            lowercased += lang.custom_case_mapping[l]
+        elif l.isupper() and \
+             lang.case_mapping and \
+             len(unicodedata.normalize('NFC', l.lower())) == 1:
+            lowercased += l.lower()
+        else:
+            lowercased += l
+    return lowercased
+
+if lang.alphabet is not None:
+    # Allowing to provide an alphabet in string format rather than list.
+    lang.alphabet = list(lang.alphabet)
+    if lang.use_ascii:
+        lang.alphabet += [chr(l) for l in range(65, 91)] + [chr(l) for l in range(97, 123)]
+    if lang.case_mapping or lang.custom_case_mapping is not None:
+        lang.alphabet = [local_lowercase(l, lang) for l in lang.alphabet]
+        #alphabet = []
+        #for l in lang.alphabet:
+            #if l.isupper() and \
+               #lang.custom_case_mapping is not None and \
+               #l in lang.custom_case_mapping:
+                #alphabet.append(lang.custom_case_mapping[l])
+            #elif l.isupper() and \
+                 #lang.case_mapping and \
+                 #len(unicodedata.normalize('NFC', l.lower())) == 1:
+                #alphabet.append(l.lower())
+            #else:
+                #alphabet.append(l)
+    lang.alphabet = list(set(lang.alphabet))
+
+# Starting processing.
+wikipedia.set_lang(lang.wikipedia_code)
+
+visited_pages = []
+
+# The full list of letter characters.
+# The key is the unicode codepoint,
+# and the value is the occurrence count.
+characters = {}
+# Sequence of letters.
+# The key is the couple (char1, char2) in unicode codepoint,
+# the value is the occurrence count.
+sequences = {}
+prev_char = None
+
+def process_text(content, lang):
+    global charsets
+    global characters
+    global sequences
+    global prev_char
+
+    if lang.clean_wikipedia_content is not None:
+        content = lang.clean_wikipedia_content(content)
+    # Clean out the Wikipedia syntax for titles.
+    content = re.sub(r'(=+) *([^=]+) *\1',
+                     r'\2', content)
+    # Clean multiple spaces. Newlines and such are normalized to spaces,
+    # since they have basically a similar role in the purpose of uchardet.
+    content = re.sub(r'\s+', ' ', content)
+
+    if lang.case_mapping or lang.custom_case_mapping is not None:
+        content = local_lowercase(content, lang)
+
+    # In python 3, strings are UTF-8.
+    # Looping through them return expected characters.
+    for char in content:
+        is_letter = False
+        if ord(char) in characters:
+            characters[ord(char)] += 1
+            is_letter = True
+        else:
+            # We save the character if it is at least in one of the
+            # language encodings and its not a special character.
+            for charset in charsets:
+                # Does the character exist in the charset?
+                try:
+                    codepoint = char.encode(charset, 'ignore')
+                except LookupError:
+                    # unknown encoding. Use iconv from command line instead.
+                    try:
+                        call = subprocess.Popen(['iconv', '-f', 'UTF-8', '-t', charset],
+                                                stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+                                                stderr=subprocess.DEVNULL)
+                        if call.poll() is not None:
+                            (_, error) = call.communicate(input='')
+                            print('Error: `iconv` ended with error "{}".\n'.format(error))
+                            exit(1)
+                        (codepoint, _) = call.communicate(input=char.encode('UTF-8'))
+                    except FileNotFoundError:
+                        print('Error: "{}" is not a supported charset by python and `iconv` is not installed.\n')
+                        exit(1)
+
+                if codepoint == b'':
+                    continue
+                # ord() is said to return the unicode codepoint.
+                # But it turns out it also gives the codepoint for other
+                # charsets if I turn the string to encoded bytes first.
+                # Not sure if that is a bug or expected.
+                codepoint = ord(codepoint)
+                if charsets[charset].charmap[codepoint] == LET:
+                    characters[ord(char)] = 1
+                    is_letter = True
+                    break
+        if is_letter:
+            if prev_char is not None:
+                if (prev_char, ord(char)) in sequences:
+                    sequences[(prev_char, ord(char))] += 1
+                else:
+                    sequences[(prev_char, ord(char))] = 1
+            prev_char = ord(char)
+        else:
+            prev_char = None
+
+def visit_pages(titles, depth, lang, logfd):
+    global visited_pages
+    global options
+
+    if len(titles) == 0:
+        return
+
+    next_titles = []
+    for title in titles:
+        if options.max_page is not None and \
+           len(visited_pages) > options.max_page:
+            return
+        if title in visited_pages:
+            continue
+        visited_pages += [title]
+        try:
+            page = wikipedia.page(title)
+        except (wikipedia.exceptions.PageError,
+                wikipedia.exceptions.DisambiguationError):
+            # Let's just discard a page when I get an exception.
+            print("Discarding page {}.\n".format(title))
+            continue
+        logfd.write("\n{} (revision {})".format(title, page.revision_id))
+
+        process_text(page.content, lang)
+        try:
+            next_titles += page.links
+        except KeyError:
+            pass
+
+    if depth >= options.max_depth:
+        return
+
+    visit_pages (next_titles, depth + 1, lang, logfd)
+
+language_c = lang.name.replace('-', '_').title()
+build_log = current_dir + '/BuildLangModelLogs/Lang{}Model.log'.format(language_c)
+logfd = open(build_log, 'w')
+logfd.write('= Logs of language model for {} ({}) =\n'.format(lang.name, lang.code))
+logfd.write('\n- Generated by {}'.format(os.path.basename(__file__)))
+logfd.write('\n- Started: {}'.format(str(datetime.datetime.now())))
+logfd.write('\n- Maximum depth: {}'.format(options.max_depth))
+if options.max_page is not None:
+    logfd.write('\n- Max number of pages: {}'.format(options.max_page))
+logfd.write('\n\n== Parsed pages ==\n')
+try:
+    visit_pages(lang.start_pages, 0, lang, logfd)
+except requests.exceptions.ConnectionError:
+    print('Error: connection to Wikipedia failed. Aborting\n')
+    exit(1)
+logfd.write('\n\n== End of Parsed pages ==')
+logfd.write('\n\n- Wikipedia parsing ended at: {}\n'.format(str(datetime.datetime.now())))
+
+########### CHARACTERS ###########
+
+# Character ratios.
+ratios = {}
+n_char = len(characters)
+occurrences = sum(characters.values())
+
+logfd.write("\n{} characters appeared {} times.\n".format(n_char, occurrences))
+for char in characters:
+    ratios[char] = characters[char] / occurrences
+    #logfd.write("Character '{}' usage: {} ({} %)\n".format(chr(char),
+    #                                                       characters[char],
+    #                                                       ratios[char] * 100))
+
+sorted_ratios = sorted(ratios.items(), key=operator.itemgetter(1),
+                       reverse=True)
+# Accumulated ratios of the frequent chars.
+accumulated_ratios = 0
+
+# If there is no alphabet defined, we just use the first 64 letters, which was
+# the original default.
+# If there is an alphabet, we make sure all the alphabet characters are in the
+# frequent list, and we stop then. There may therefore be more or less than
+# 64 frequent characters depending on the language.
+if lang.alphabet is None:
+    freq_count = 64
+else:
+    freq_count = 0
+    for order, (char, ratio) in enumerate(sorted_ratios):
+        if len(lang.alphabet) == 0:
+            break
+        if chr(char) in lang.alphabet:
+            lang.alphabet.remove(chr(char))
+        freq_count += 1
+    else:
+        if len(lang.alphabet) > 0:
+            print("Error: alphabet characters are absent from data collection"
+                  "\n       Please check the configuration or the data."
+                  "\n       Missing characters: {}".format(", ".join(lang.alphabet)))
+            exit(1)
+
+logfd.write('\nFirst {} characters:'.format(freq_count))
+for order, (char, ratio) in enumerate(sorted_ratios):
+    if order >= freq_count:
+        break
+    logfd.write("\n[{:2}] Char {}: {} %".format(order, chr(char), ratio * 100))
+    accumulated_ratios += ratio
+
+logfd.write("\n\nThe first {} characters have an accumulated ratio of {}.\n".format(freq_count, accumulated_ratios))
+
+with open(current_dir + '/header-template.cpp', 'r') as header_fd:
+    c_code = header_fd.read()
+
+c_code += '\n/********* Language model for: {} *********/\n\n'.format(lang.name)
+c_code += '/**\n * Generated by {}\n'.format(os.path.basename(__file__))
+c_code += ' * On: {}\n'.format(str(datetime.datetime.now()))
+c_code += ' **/\n'
+
+c_code += \
+"""
+/* Character Mapping Table:
+ * ILL: illegal character.
+ * CTR: control character specific to the charset.
+ * RET: carriage/return.
+ * SYM: symbol (punctuation) that does not belong to word.
+ * NUM: 0 - 9.
+ *
+ * Other characters are ordered by probabilities
+ * (0 is the most common character in the language).
+ *
+ * Orders are generic to a language. So the codepoint with order X in
+ * CHARSET1 maps to the same character as the codepoint with the same
+ * order X in CHARSET2 for the same language.
+ * As such, it is possible to get missing order. For instance the
+ * ligature of 'o' and 'e' exists in ISO-8859-15 but not in ISO-8859-1
+ * even though they are both used for French. Same for the euro sign.
+ */
+"""
+
+for charset in charsets:
+    charset_c = charset.replace('-', '_').title()
+    CTOM_str = 'static const unsigned char {}_CharToOrderMap[]'.format(charset_c)
+    CTOM_str += ' =\n{'
+    for line in range(0, 16):
+        CTOM_str += '\n  '
+        for column in range(0, 16):
+            cp = line * 16 + column
+            cp_type = charsets[charset].charmap[cp]
+            if cp_type == ILL:
+                CTOM_str += 'ILL,'
+            elif cp_type == RET:
+                CTOM_str += 'RET,'
+            elif cp_type == CTR:
+                CTOM_str += 'CTR,'
+            elif cp_type == SYM:
+                CTOM_str += 'SYM,'
+            elif cp_type == NUM:
+                CTOM_str += 'NUM,'
+            else: # LET
+                try:
+                    uchar = bytes([cp]).decode(charset)
+                except UnicodeDecodeError:
+                    print('Unknown character 0X{:X} in {}.'.format(cp, charset))
+                    print('Please verify your charset specification.\n')
+                    exit(1)
+                except LookupError:
+                    # Unknown encoding. Use iconv instead.
+                    try:
+                        call = subprocess.Popen(['iconv', '-t', 'UTF-8', '-f', charset],
+                                                stdin=subprocess.PIPE,
+                                                stdout=subprocess.PIPE,
+                                                stderr=subprocess.PIPE)
+                        if call.poll() is not None:
+                            (_, error) = call.communicate(input='')
+                            print('Error: `iconv` ended with error "{}".\n'.format(error))
+                            exit(1)
+                        (uchar, _) = call.communicate(input=bytes([cp]))
+                        uchar = uchar.decode('UTF-8')
+                    except FileNotFoundError:
+                        print('Error: "{}" is not a supported charset by python and `iconv` is not installed.\n')
+                        exit(1)
+                #if lang.case_mapping and uchar.isupper() and \
+                   #len(unicodedata.normalize('NFC', uchar.lower())) == 1:
+                   # Unless we encounter special cases of characters with no
+                   # composed lowercase, we lowercase it.
+                if lang.case_mapping or lang.custom_case_mapping is not None:
+                    uchar = local_lowercase(uchar, lang)
+                for order, (char, ratio) in enumerate(sorted_ratios):
+                    if char == ord(uchar):
+                        CTOM_str += '{:3},'.format(min(249, order))
+                        break
+                else:
+                    # XXX: we must make sure the character order does not go
+                    # over the special characters (250 currently). This may
+                    # actually happen when building a model for a language
+                    # writable with many different encoding. So let's just
+                    # ceil the order value at 249 max.
+                    # It may be an interesting alternative to add another
+                    # constant for any character with an order > freqCharCount.
+                    # Maybe IRR (irrelevant character) or simply CHR.
+                    CTOM_str += '{:3},'.format(min(249, n_char))
+                    n_char += 1
+        CTOM_str += ' /* {:X}X */'.format(line)
+    CTOM_str += '\n};\n/*'
+    CTOM_str += 'X0  X1  X2  X3  X4  X5  X6  X7  X8  X9  XA  XB  XC  XD  XE  XF'
+    CTOM_str += ' */\n\n'
+    c_code += CTOM_str
+
+########### SEQUENCES ###########
+
+ratios = {}
+occurrences = sum(sequences.values())
+ratio_512 = 0
+ratio_1024 = 0
+
+sorted_seqs = sorted(sequences.items(), key=operator.itemgetter(1),
+                     reverse=True)
+for order, ((c1, c2), count) in enumerate(sorted_seqs):
+    if order < 512:
+        ratio_512 += count
+    elif order < 1024:
+        ratio_1024 += count
+    else:
+        break
+ratio_512 /= occurrences
+ratio_1024 /= occurrences
+
+logfd.write("\n{} sequences found.\n".format(len(sorted_seqs)))
+
+c_code += """
+/* Model Table:
+ * Total sequences: {}
+ * First 512 sequences: {}
+ * Next 512 sequences (512-1024): {}
+ * Rest: {}
+ * Negative sequences: TODO""".format(len(sorted_seqs),
+                                      ratio_512,
+                                      ratio_1024,
+                                      1 - ratio_512 - ratio_1024)
+
+logfd.write("\nFirst 512 (typical positive ratio): {}".format(ratio_512))
+logfd.write("\nNext 512 (512-1024): {}".format(ratio))
+logfd.write("\nRest: {}".format(1 - ratio_512 - ratio_1024))
+
+c_code += "\n */\n"
+
+LM_str = 'static const PRUint8 {}LangModel[]'.format(language_c)
+LM_str += ' =\n{'
+for line in range(0, freq_count):
+    LM_str += '\n  '
+    for column in range(0, freq_count):
+        # Let's not make too long lines.
+        if freq_count > 40 and column == int(freq_count / 2):
+            LM_str += '\n   '
+        first_order = int(line)
+        second_order = column
+        if first_order < len(sorted_ratios) and second_order < len(sorted_ratios):
+            (first_char, _) = sorted_ratios[first_order]
+            (second_char, _) = sorted_ratios[second_order]
+            if (first_char, second_char) in sequences:
+                for order, (seq, _) in enumerate(sorted_seqs):
+                    if seq == (first_char, second_char):
+                        if order < 512:
+                            LM_str += '3,'
+                        elif order < 1024:
+                            LM_str += '2,'
+                        else:
+                            LM_str += '1,'
+                        break
+                else:
+                    pass # impossible!
+                    LM_str += '0,'
+            else:
+                LM_str += '0,'
+        else:
+            # It may indeed happen that we find less than 64 letters used for a
+            # given language.
+            LM_str += '0,'
+LM_str += '\n};\n'
+c_code += LM_str
+
+for charset in charsets:
+    charset_c = charset.replace('-', '_').title()
+    SM_str = '\n\nconst SequenceModel {}{}Model ='.format(charset_c, language_c)
+    SM_str += '\n{\n  '
+    SM_str += '{}_CharToOrderMap,\n  {}LangModel,'.format(charset_c, language_c)
+    SM_str += '\n  {},'.format(freq_count)
+    SM_str += '\n  (float){},'.format(ratio_512)
+    SM_str += '\n  {},'.format('PR_TRUE' if lang.use_ascii else 'PR_FALSE')
+    SM_str += '\n  "{},"'.format(charset)
+    SM_str += '\n  "{}"'.format(lang.code)
+    SM_str += '\n};'
+    c_code += SM_str
+
+c_code += '\n'
+
+lang_model_file = current_dir + '/../src/LangModels/Lang{}Model.cpp'.format(language_c)
+with open(lang_model_file, 'w') as cpp_fd:
+    cpp_fd.write(c_code)
+
+logfd.write('\n\n- Processing end: {}\n'.format(str(datetime.datetime.now())))
+logfd.close()
+
+print("The following language model file has been generated: {}"
+      "\nThe build log is available in: {}"
+      "\nTest them and commit them.".format(lang_model_file, build_log))