subconv 0.1.0 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 28cd6dd4e5675ee424b07b6ad013fb694e9145b8
-  data.tar.gz: b81878db9e8705adf103bd0ecd40466e54512f77
+SHA256:
+  metadata.gz: 6348b3b43fd4b191b25d166007dd539d18d3e72c13f32cb23e7feb839c06d665
+  data.tar.gz: 8a0b8177d72cf42ff45649e28d27c333ad223072193181bcb023a6349c8e6ee5
 SHA512:
-  metadata.gz: a7ac077fc7d2691d74141d49e99051b0679202ac40625f8150846c0f625db85614e636a866db1916b465290143dae41549dd4b98f4712bffd4e11e2c6a7b4995
-  data.tar.gz: 646035c2eaaa3e9726621f552fdd44c0b27b56733f6a7dd70c4677e21fd72893e2748177727e027a6ab5719d9a8afc5ca0da0926f9d278922ed501431e034080
+  metadata.gz: d805a1a29e705a9fbc23ec481484600f99ef844fc51422469a1768d255d2894e8c7c06fe9c45079cf97b99a86b5bbb34df0c7c09083ac5cd493d084ebce605a7
+  data.tar.gz: bfbe89e2915ffc77484521d8e291cb8f58685dc8e5bf819c34c62bb8344b03c187dfcedce55363003ab1565d7eb59694f4cee1aee85c2a9e9b9cee84cb1d6ac6

data/.gitignore

@@ -30,7 +30,7 @@ build/
 
 # for a library or gem, you might want to ignore these files since the code is
 # intended to run in multiple environments; otherwise, check them in:
-Gemfile.lock
+#Gemfile.lock
 
 # .ruby-version
 # .ruby-gemset

data/.gitlab-ci.yml

@@ -0,0 +1,33 @@
+before_script:
+  - apt-get update -qq
+  - ruby -v
+  - which ruby
+  - gem install bundler --no-document --version "~>1.0"
+  - bundle install --jobs $(nproc)  "${FLAGS[@]}"
+
+rspec:
+  script:
+    - bundle exec rake spec
+
+rubocop:
+  script:
+    - bundle exec rake rubocop
+
+dependency_scanning:
+  before_script: []
+  image: docker:stable
+  variables:
+    DOCKER_DRIVER: overlay2
+  allow_failure: true
+  services:
+    - docker:stable-dind
+  script:
+    - export SP_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')
+    - docker run
+        --env DEP_SCAN_DISABLE_REMOTE_CHECKS="${DEP_SCAN_DISABLE_REMOTE_CHECKS:-false}"
+        --volume "$PWD:/code"
+        --volume /var/run/docker.sock:/var/run/docker.sock
+        "registry.gitlab.com/gitlab-org/security-products/dependency-scanning:$SP_VERSION" /code
+  artifacts:
+    reports:
+      dependency_scanning: gl-dependency-scanning-report.json

data/.rubocop.yml

@@ -1,20 +1,14 @@
 AllCops:
-  TargetRubyVersion: 2.2
+  TargetRubyVersion: 2.4
   Exclude:
     - '**/bin/**/*'
-    - '**/db/schema.rb'
     - '**/config/**/*'
     - '**/lib/tasks/*.rake'
+    - 'vendor/**/*'
 
-# Use only ascii symbols in identifiers.
-AsciiComments:
-  Enabled: false
-
-# Document classes and non-namespace modules.
 Documentation:
   Enabled: false
 
-# Limit lines to 79 characters.
 LineLength:
   Enabled: false
 
@@ -24,17 +18,17 @@ BlockDelimiters:
 MethodLength:
   Max: 30
 
-# Checks for proper usage of fail and raise.
-#SignalException:
-#  Enabled: false
+UncommunicativeMethodParamName:
+  AllowedNames:
+    - hi
+    - io
+    - lo
+    - x
+    - y
 
 StringLiterals:
   EnforcedStyle: single_quotes
 
-# Avoid the use of attr. Use attr_reader and attr_accessor instead.
-Attr:
-  Enabled: false
-
 UnusedBlockArgument:
   AutoCorrect: false
 
@@ -44,31 +38,33 @@ UnusedMethodArgument:
 CyclomaticComplexity:
   Max: 30
 
-# https://github.com/bbatsov/ruby-style-guide/issues/395
-RaiseArgs:
-  Enabled: false
-
 AbcSize:
   Max: 20
 
-TrivialAccessors:
-  AllowPredicates: true
+
+# Indentation
 
 AlignHash:
   EnforcedHashRocketStyle: table
   EnforcedColonStyle: table
 
-IndentArray:
+IndentFirstArrayElement:
+  EnforcedStyle: consistent
+
+IndentFirstHashElement:
   EnforcedStyle: consistent
 
 SignalException:
   EnforcedStyle: semantic
 
-#MultilineMethodCallIndentation:
-#  EnforcedStyle: indented
-#
-#MultilineOperationIndentation:
-#  EnforcedStyle: indented
-#
-#AlignParameters:
-#  Enabled: false
+IndentFirstArgument:
+  EnforcedStyle: consistent
+
+MultilineMethodCallIndentation:
+  EnforcedStyle: indented
+
+MultilineOperationIndentation:
+  EnforcedStyle: indented
+
+AlignParameters:
+  EnforcedStyle: with_fixed_indentation

data/Gemfile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 source 'https://rubygems.org'
 
 gemspec

data/Gemfile.lock

@@ -0,0 +1,73 @@
+PATH
+  remote: .
+  specs:
+    subconv (1.0.0)
+      solid-struct (~> 0.1)
+      timecode (~> 2.2)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    ansi (1.5.0)
+    approximately (1.1.0)
+    ast (2.4.0)
+    diff-lcs (1.3)
+    docile (1.3.2)
+    jaro_winkler (1.5.3)
+    json (2.2.0)
+    parallel (1.17.0)
+    parser (2.6.3.0)
+      ast (~> 2.4.0)
+    rainbow (3.0.0)
+    rake (12.3.2)
+    rspec (3.8.0)
+      rspec-core (~> 3.8.0)
+      rspec-expectations (~> 3.8.0)
+      rspec-mocks (~> 3.8.0)
+    rspec-core (3.8.2)
+      rspec-support (~> 3.8.0)
+    rspec-expectations (3.8.4)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-mocks (3.8.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-support (3.8.2)
+    rubocop (0.72.0)
+      jaro_winkler (~> 1.5.1)
+      parallel (~> 1.10)
+      parser (>= 2.6)
+      rainbow (>= 2.2.2, < 4.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 1.7)
+    ruby-progressbar (1.10.1)
+    simplecov (0.17.0)
+      docile (~> 1.1)
+      json (>= 1.8, < 3)
+      simplecov-html (~> 0.10.0)
+    simplecov-console (0.5.0)
+      ansi
+      simplecov
+      terminal-table
+    simplecov-html (0.10.2)
+    solid-struct (0.1.0)
+    terminal-table (1.8.0)
+      unicode-display_width (~> 1.1, >= 1.1.1)
+    timecode (2.2.2)
+      approximately (~> 1.1)
+    unicode-display_width (1.6.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.7)
+  rake (~> 12.3)
+  rspec (~> 3.8.0)
+  rubocop (~> 0.72)
+  simplecov (~> 0.17)
+  simplecov-console (~> 0.5)
+  subconv!
+
+BUNDLED WITH
+   1.16.1

data/README.md

@@ -1,9 +1,3 @@
-[![Gem Version](https://badge.fury.io/rb/subconv.svg)](http://badge.fury.io/rb/subconv)
-[![Dependency Status](https://gemnasium.com/pkerling/subconv.svg)](https://gemnasium.com/pkerling/subconv)
-[![Build Status](https://travis-ci.org/pkerling/subconv.svg?branch=master)](https://travis-ci.org/pkerling/subconv)
-[![Coverage Status](https://coveralls.io/repos/github/pkerling/subconv/badge.svg?branch=master)](https://coveralls.io/github/pkerling/subconv?branch=master)
-[![Inline docs](http://inch-ci.org/github/pkerling/subconv.svg?branch=master)](http://inch-ci.org/github/pkerling/subconv)
-
 subconv - Ruby SCC (EIA-608) to WebVTT subtitle converter
 =========================================================
 
@@ -26,6 +20,7 @@ Usage
 	    -c, --no-color                   Remove all color information from output
 	    -F, --no-flash                   Remove all flash (blinking) information from output
 	    -s, --simple-positions           Convert to simple top/bottom center-aligned captions
+	    -p, --keep-paint-on              Keep paint-on captions, i.e. do not combine them into pop-on cues
 	    -h, --help                       Show this help message and quit.
 
 The API can also be used programmatically, the `bin/subconv` file is just an
@@ -35,12 +30,15 @@ Supported features
 ------------------
 * EIA-608 parsing and conversion to WebVTT
   * Pop-on captions
+  * Paint-on captions
   * Full positioning
-  * All special characters defined in the standard
+  * All special characters defined in the FCC standard
+  * All extended (Western European) characters defined in EIA-608-B
   * All colors
   * Italics
   * Underline
   * Flash
+* Paint-on to pop-on caption conversion (by combining similar cues)
 * Optional removal of certain features during the conversion
   * Color
   * Flash
@@ -55,5 +53,4 @@ Firefox can not apply classes/styles inside cues at all.
 Unsupported features
 --------------------
 * Roll-up captions
-* Paint-on captions
 * EIA-708

data/Rakefile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'bundler/gem_tasks'
 require 'rspec/core/rake_task'
 require 'rubocop/rake_task'
@@ -7,4 +9,4 @@ RuboCop::RakeTask.new do |task|
   task.options = ['--fail-level', 'warning']
 end
 
-task default: [:spec, :rubocop]
+task default: %i[spec rubocop]

data/bin/subconv

@@ -21,6 +21,9 @@ OptionParser.new do |opts|
   opts.on('-s', '--simple-positions', 'Convert to simple top/bottom center-aligned captions') do
     options[:simple_positions] = true
   end
+  opts.on('-p', '--keep-paint-on', 'Keep paint-on captions, i.e. do not combine them into pop-on cues') do
+    options[:keep_paint_on] = true
+  end
   opts.on_tail('-h', '--help', 'Show this help message and quit.') do
     puts opts.help
     exit
@@ -35,7 +38,10 @@ reader = Subconv::Scc::Reader.new
 File.open(options[:in_file], File::RDONLY) do |file|
   reader.read(file, options[:fps].to_f)
 end
-captions = Subconv::Scc::Transformer.new.transform reader.captions
+transformer = Subconv::Scc::Transformer.new
+scc_captions = reader.captions
+scc_captions = transformer.combine_paint_on_captions(scc_captions) unless options[:keep_paint_on]
+captions = transformer.transform scc_captions
 filter_options = {}
 filter_options[:remove_color] = true if options[:no_color]
 filter_options[:remove_flash] = true if options[:no_flash]

data/lib/subconv.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'subconv/version'
 require 'subconv/scc/reader'
 require 'subconv/scc/transformer'

data/lib/subconv/caption.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Subconv
   # Two-dimensional screen position relative (both x and y position between 0 and 1) to the screen size
   class Position
@@ -17,6 +19,7 @@ module Subconv
     def x=(x)
       x = x.to_f
       fail RangeError, 'X position not between 0 and 1' unless x.between?(0.0, 1.0)
+
       @x = x
     end
 
@@ -24,6 +27,7 @@ module Subconv
     def y=(y)
       y = y.to_f
       fail RangeError, 'Y position not between 0 and 1' unless y.between?(0.0, 1.0)
+
       @y = y
     end
   end
@@ -58,6 +62,7 @@ module Subconv
 
     def children=(children)
       fail 'Children must be an array' unless children.class == Array
+
       @children = children
     end
   end

data/lib/subconv/caption_filter.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'subconv/caption'
 
 module Subconv
@@ -65,9 +67,7 @@ module Subconv
       # Remove nils resulting from removed captions
       captions.compact!
       # Do per-caption processing after merging etc.
-      captions.each do |caption|
-        process_caption!(caption)
-      end
+      captions.each(&method(:process_caption!))
     end
 
     def process_caption!(caption)
@@ -88,7 +88,9 @@ module Subconv
           if current_text_node.nil?
             current_text_node = child
           else
-            # Add text to previous node
+            # Add text to previous node, copying the text if necessary (literals are frozen by default)
+            current_text_node.text = current_text_node.text.dup if current_text_node.text.frozen?
+
             current_text_node.text << child.text
             # Remove this node
             next

data/lib/subconv/scc/reader.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require 'subconv/utility'
 
 require 'solid_struct'
@@ -6,7 +7,7 @@ require 'timecode'
 
 module Subconv
   module Scc
-    FILE_MAGIC = 'Scenarist_SCC V1.0'.freeze
+    FILE_MAGIC = 'Scenarist_SCC V1.0'
 
     # Grid size
     GRID_ROWS = 15
@@ -14,8 +15,18 @@ module Subconv
 
     # Grid is just an array with some extra convenience functions and a default size
     class Grid < Array
-      def initialize
-        super(GRID_ROWS) { Array.new(GRID_COLUMNS) }
+      def initialize(*args)
+        if args.empty?
+          super(GRID_ROWS) { Array.new(GRID_COLUMNS) }
+        elsif args.size == 1
+          fail ArgumentError, 'Attempted to create grid from unsupported type' unless args[0].is_a?(Array)
+          fail ArgumentError, 'Grid has illegal row count' if args[0].size != GRID_ROWS
+          fail ArgumentError, 'Grid has illegal column count' if args[0].any? { |row| row.size != GRID_COLUMNS }
+
+          super(args.first)
+        else
+          fail ArgumentError, 'Illegal number of parameters'
+        end
       end
 
       # The grid is empty when there are no characters in it
@@ -23,6 +34,11 @@ module Subconv
         flatten.compact.empty?
       end
 
+      # Make a deep copy
+      def clone
+        Grid.new(map { |row| row.map(&:clone) })
+      end
+
       # Insert continuous text at a given position
       # Returns self for chaining
       def insert_text(row, column, text, style = CharacterStyle.default)
@@ -32,6 +48,21 @@ module Subconv
         end
         self
       end
+
+      def without_identical_characters(other_grid)
+        result = clone
+        each_with_index do |row, row_i|
+          row.each_with_index do |column, column_i|
+            result[row_i][column_i] = nil if other_grid[row_i][column_i] == column
+          end
+        end
+
+        result
+      end
+
+      def to_simple_text
+        map { |row| row.map { |char| char.nil? ? ' ' : char.character }.join }.join("\n")
+      end
     end
 
     # Color constants as immutable value objects with some convenience functions (e.g. conversion to string or symbol)
@@ -92,6 +123,7 @@ module Subconv
       def self.for_value(value)
         color = COLORS[value]
         fail "Color value #{value} is unknown" if color.nil?
+
         color
       end
 
@@ -111,7 +143,45 @@ module Subconv
     Character = SolidStruct.new(:character, :style)
 
     # One fully rendered caption displayed at a specific point in time
-    Caption = SolidStruct.new(:timecode, :grid)
+    class Caption
+      def initialize(args)
+        # Inject default
+        args = { mode: :pop_on, char_replacement: false }.merge(args)
+        self.timecode = args[:timecode]
+        self.grid = args[:grid]
+        self.mode = args[:mode]
+        self.char_replacement = args[:char_replacement]
+      end
+
+      def ==(other)
+        timecode == other.timecode && grid == other.grid && mode == other.mode && char_replacement? == other.char_replacement?
+      end
+
+      alias eql? ==
+
+      attr_reader :mode
+      attr_writer :char_replacement
+      attr_accessor :timecode, :grid
+
+      def mode=(mode)
+        fail 'Unknown mode' unless %i[pop_on paint_on].include?(mode)
+
+        @mode = mode
+      end
+
+      def pop_on_mode?
+        @mode == :pop_on
+      end
+
+      def paint_on_mode?
+        @mode == :paint_on
+      end
+
+      # Is this caption just replacing an existing character with an extended character?
+      def char_replacement?
+        @char_replacement
+      end
+    end
 
     # SCC reader
     # Parse and render an SCC file sequentially into a background and foreground grid
@@ -122,23 +192,24 @@ module Subconv
     # The advanced recovery methods mentioned in CEA608 are not implemented since the source is assumed to contain no errors (e.g. DVD source).
     class Reader
       # Regular expression for parsing one line of data
-      LINE_REGEXP = /^(?<timecode>[0-9:;]+)\t(?<data>(?:[0-9a-fA-F]{4} ?)+)$/
+      LINE_REGEXP = /^(?<timecode>[0-9:;]+)\t(?<data>(?:[0-9a-fA-F]{4} ?)+)$/.freeze
 
       # rubocop:disable MutableConstant
 
       # Map of standard characters that do not match the standard ASCII codes
       # to their corresponding unicode characters
       STANDARD_CHARACTER_MAP = {
-        '*'    => "\u00e1",
-        '\\'   => "\u00e9",
-        '^'    =>	"\u00ed",
-        '_'    =>	"\u00f3",
-        '`'    =>	"\u00fa",
-        '{'    =>	"\u00e7",
-        '|'    =>	"\u00f7",
-        '}'    =>	"\u00d1",
-        '~'    =>	"\u00f1",
-        "\x7f" =>	"\u2588"
+        '\''   => '’', # CEA-608-B says to use curly apostrophe
+        '*'    => 'á',
+        '\\'   => 'é',
+        '^'    =>	'í',
+        '_'    =>	'ó',
+        '`'    =>	'ú',
+        '{'    =>	'ç',
+        '|'    =>	'÷',
+        '}'    =>	'Ñ',
+        '~'    =>	'ñ',
+        "\x7f" =>	'█'
       }
       # rubocop:enable MutableConstant
       # Simply return the character if no exception matched
@@ -148,24 +219,14 @@ module Subconv
       STANDARD_CHARACTER_MAP.freeze
 
       # Map of special characters to unicode codepoints
-      SPECIAL_CHARACTER_MAP = {
-        '0' => "\u00ae",
-        '1' => "\u00b0",
-        '2' => "\u00bd",
-        '3' => "\u00bf",
-        '4' => "\u2122",
-        '5' => "\u00a2",
-        '6' => "\u00a3",
-        '7' => "\u266a",
-        '8' => "\u00e0",
-        # "\x39" => 	transparent space is handled specially since it is not a real character
-        ':' => "\u00e8",
-        ';' => "\u00e2",
-        '<' => "\u00ea",
-        '=' => "\u00ee",
-        '>' => "\u00f4",
-        '?' => "\u00fb"
-      }.freeze
+      # 0x39 transparent space is handled specially since it is not a real character
+      SPECIAL_CHARACTER_MAP = Hash[%w[® ° ½ ¿ ™ ¢ £ ♪ à _ è á ê î ô û].each_with_index.map { |char, i| [0x30 + i, char] }].freeze
+
+      # Extended character maps for Western European languages and box drawing
+      EXTENDED_CHARACTER_MAPS = [
+        Hash[%w[Á É Ó Ú Ü ü ‘ ¡ * ' ─ © ℠ · “ ” À Â Ç È Ê Ë ë Î Ï ï Ô Ù ù Û « »].each_with_index.map { |char, i| [0x20 + i, char] }].freeze,
+        Hash[%w[Ã ã Í Ì ì Ò ò Õ õ { } \\ ^ _ | ~ Ä ä Ö ö ß ¥ ¤ │ Å å Ø ø ┌ ┐ └ ┘].each_with_index.map { |char, i| [0x20 + i, char] }].freeze
+      ].freeze
 
       # Map of preamble address code high bytes to their
       # corresponding base row numbers (counted from 0)
@@ -185,16 +246,17 @@ module Subconv
       class InvalidFormatError < Error; end
       class ParityError < Error; end
 
-      # Internal state of the parser consisting of current drawing position and character style
+      # Internal state of the parser consisting of current drawing position, character style and captioning mode
       class State
         def initialize(params)
           self.row = params[:row]
           self.column = params[:column]
           @style = params[:style]
+          self.mode = params[:mode]
         end
 
         attr_accessor :style
-        attr_reader :row, :column
+        attr_reader :row, :column, :mode, :char_replaced
 
         # Make sure the maximum row count is not exceeded
         def row=(row)
@@ -206,8 +268,32 @@ module Subconv
           @column = Utility.clamp(column, 0, GRID_COLUMNS - 1)
         end
 
+        def mode=(mode)
+          fail 'Unknown mode' unless %i[pop_on paint_on].include?(mode)
+
+          @mode = mode
+        end
+
+        def pop_on_mode?
+          @mode == :pop_on
+        end
+
+        def paint_on_mode?
+          @mode == :paint_on
+        end
+
+        def char_replaced=(replaced)
+          fail ArgumentError, 'Invalid value for char_replaced' unless [true, false].include?(replaced)
+
+          @char_replaced = replaced
+        end
+
+        def start_new_frame
+          self.char_replaced = false
+        end
+
         def self.default
-          State.new(row: 0, column: 0, style: CharacterStyle.default)
+          State.new(row: 0, column: 0, style: CharacterStyle.default, mode: :pop_on)
         end
       end
 
@@ -225,6 +311,8 @@ module Subconv
         @now = Timecode.new(0, fps)
         @data_channel = 0
 
+        update_active_grid
+
         magic = io.readline.chomp!
         fail InvalidFormatError, 'File does not start with "' + Scc::FILE_MAGIC + '"' unless Scc::FILE_MAGIC == magic
 
@@ -235,6 +323,7 @@ module Subconv
 
           line_data = LINE_REGEXP.match(line)
           fail InvalidFormatError, "Invalid line \"#{line}\"" if line_data.nil?
+
           # Parse timecode
           old_time = @now
           timecode = Timecode.new(line_data[:timecode], fps)
@@ -254,10 +343,13 @@ module Subconv
 
         data.split(' ').each do |word_string|
           begin
+            @state.start_new_frame
+
             # Decode hexadecimal word into two-byte string
             word = [word_string].pack('H*')
             # Check parity
             fail ParityError, "At least one byte in word #{word_string} has even parity, odd required" unless !check_parity || (correct_parity?(word[0]) && correct_parity?(word[1]))
+
             # Remove parity bit for further processing
             word = word.bytes.collect { |byte|
               # Unset 8th bit
@@ -298,15 +390,20 @@ module Subconv
                 # for further processing.
               end
 
-              if hi == 0x11 && lo >= 0x30 && lo <= 0x3f
+              # rubocop:disable Style/NumericPredicate
+
+              if hi == 0x11 && (0x30..0x3f).cover?(lo)
                 # Special character
                 handle_special_character(lo)
-              elsif hi >= 0x10 && hi <= 0x17 && lo >= 0x40
-                # Premable address code
+              elsif (0x12..0x13).cover?(hi) && (0x20..0x3f).cover?(lo)
+                # Extended character
+                handle_extended_character(hi & 1, lo)
+              elsif (0x10..0x17).cover?(hi) && (0x40..0xff).cover?(lo)
+                # Preamble address code
                 handle_preamble_address_code(hi, lo)
-              elsif (hi == 0x14 || hi == 0x17) && lo >= 0x20 && lo <= 0x2f
+              elsif [0x14, 0x17].include?(hi) && (0x20..0x2f).cover?(lo)
                 handle_control_code(hi, lo)
-              elsif hi == 0x11 && lo >= 0x20 && lo <= 0x2f
+              elsif hi == 0x11 && (0x20..0x2f).cover?(lo)
                 handle_mid_row_code(hi, lo)
               elsif hi == 0x00 && lo == 0x00
                 # Ignore filler
@@ -314,9 +411,12 @@ module Subconv
                 puts "Ignoring unknown command #{hi}/#{lo}"
               end
 
+              # rubocop:enable Style/NumericPredicate
+
               last_command = word
             end
 
+            post_frame if @state.paint_on_mode?
           ensure
             # Advance one frame for each word read
             @now += 1
@@ -327,14 +427,14 @@ module Subconv
       # Insert one unicode character into the grid at the current position and with the
       # current style, then advance the cursor one column
       def insert_character(char)
-        @background_grid[@state.row][@state.column] = Character.new(char, @state.style.dup)
+        @active_grid[@state.row][@state.column] = Character.new(char, @state.style.dup)
         @state.column += 1
       end
 
       # Insert a CEA608 character into the grid at the current position, converting it to its unicode representation
       def handle_character(byte)
         # Ignore filler character
-        return if byte == 0
+        return if byte.zero?
 
         char = STANDARD_CHARACTER_MAP[byte.chr]
         insert_character(char)
@@ -345,17 +445,26 @@ module Subconv
       def handle_special_character(byte)
         if byte == 0x39
           # Transparent space: Move cursor after deleting the current column to open up a hole
-          @background_grid[@state.row][@state.column] = nil
+          @active_grid[@state.row][@state.column] = nil
           @state.column += 1
         else
-          char = SPECIAL_CHARACTER_MAP[byte.chr]
+          char = SPECIAL_CHARACTER_MAP.fetch(byte)
           insert_character(char)
         end
       end
 
+      # Insert an extended character into the grid at the current position, converting it to its Unicode representation
+      def handle_extended_character(map, byte)
+        char = EXTENDED_CHARACTER_MAPS.fetch(map).fetch(byte)
+        # Extended characters include automatic backspace+overwrite
+        @state.column -= 1
+        insert_character(char)
+        @state.char_replaced = true
+      end
+
       # Set drawing position and style according to the information in a preamble address code
       def handle_preamble_address_code(hi, lo)
-        @state.row = PREAMBLE_ADDRESS_CODE_ROW_MAP[hi]
+        @state.row = PREAMBLE_ADDRESS_CODE_ROW_MAP.fetch(hi)
         # Low byte bit 5 adds 1 to the row number if set
         @state.row += 1 if lo & (1 << 5) != 0
 
@@ -375,12 +484,16 @@ module Subconv
           @state.style.color = Color::WHITE
           # One indent equals 4 characters
           @state.column = color_or_indent * 4
-        elsif color_or_indent == 7
-          # "color" 7 is white with italics
-          @state.style.color = Color::WHITE
-          @state.style.italics = true
         else
-          @state.style.color = Color.for_value(color_or_indent)
+          # Style code always sets first column
+          @state.column = 0
+          if color_or_indent == 7
+            # "color" 7 is white with italics
+            @state.style.color = Color::WHITE
+            @state.style.italics = true
+          else
+            @state.style.color = Color.for_value(color_or_indent)
+          end
         end
       end
 
@@ -388,38 +501,49 @@ module Subconv
       def handle_control_code(hi, lo)
         if hi == 0x14 && lo == 0x20
           # Resume caption loading
-          # Nothing to do here, only pop-onstyle is supported anyway
+          @state.mode = :pop_on
+          update_active_grid
         elsif hi == 0x14 && lo == 0x21
           # Backspace
           unless @state.column.zero? # Ignore in the first column
             @state.column -= 1
             # Delete character at cursor after moving one character back
-            @background_grid[@state.row][@state.column] = nil
+            @active_grid[@state.row][@state.column] = nil
           end
         elsif hi == 0x14 && lo == 0x24
           # Delete to end of row
           (@state.column...GRID_COLUMNS).each do |column|
-            @background_grid[@state.row][column] = nil
+            @active_grid[@state.row][column] = nil
           end
         elsif hi == 0x14 && lo == 0x28
           # Flash on
           # Flash is a spacing character
           insert_character(' ')
           @state.style.flash = true
+        elsif hi == 0x14 && lo == 0x29
+          # Resume direct captioning
+          @state.mode = :paint_on
+          update_active_grid
           # elsif hi == 0x14 && lo == 0x2b
-          # Resume text display -> not a pop-on command
+          # Resume text display -> command not described in spec
           # fail "RTD"
         elsif hi == 0x14 && lo == 0x2c
           # Erase displayed memory
           @foreground_grid = Grid.new
           post_frame
+          update_active_grid
         elsif hi == 0x14 && lo == 0x2e
           # Erase non-displayed memory
           @background_grid = Grid.new
+          update_active_grid
         elsif hi == 0x14 && lo == 0x2f
           # End of caption (flip memories)
           @foreground_grid, @background_grid = @background_grid, @foreground_grid
+          # This also forces pop-on mode
+          @state.mode = :pop_on
           post_frame
+
+          update_active_grid
         elsif hi == 0x17 && lo >= 0x21 && lo <= 0x23
           # Tab offset
           # Bits 0 and 1 designate how many columns to go
@@ -452,13 +576,17 @@ module Subconv
       # Insert the currently displayed foreground grid as caption into the captions array
       # Must be called whenever the foreground grid is changed as a result of a command
       def post_frame
-        # Only push a new caption if the grid has changed
-        if @captions.empty? || @foreground_grid != @last_grid
-          # Save space by not saving the grid if it is completely empty
-          grid = @foreground_grid.empty? ? nil : @foreground_grid
-          @captions.push(Caption.new(timecode: @now, grid: grid))
-          @last_grid = @foreground_grid
-        end
+        # Only push a new caption if the grid has changed, but do not push out an empty grid initially
+        return unless @foreground_grid != @last_grid && !(@last_grid.nil? && @foreground_grid.empty?)
+
+        # Save space by not saving the grid if it is completely empty
+        grid = @foreground_grid.empty? ? nil : @foreground_grid
+        @captions.push(Caption.new(timecode: @now, grid: grid.clone, mode: @state.mode, char_replacement: @state.char_replaced))
+        @last_grid = @foreground_grid.clone
+      end
+
+      def update_active_grid
+        @active_grid = @state.paint_on_mode? ? @foreground_grid : @background_grid
       end
 
       # Check a byte for odd parity