ruby-sfst 0.3.1 → 0.4.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 33ebe15e44bf43836a7b0f40b47ea75a9203a45f
+  data.tar.gz: 12ae1d808be8c3602a72191138cff2da664fc54c
+SHA512:
+  metadata.gz: 4fce794f851b888960202b2d624d9ebcee447a5fe396e600826412ea2bf0859dd7ced63fe3f472b8deeb51cae5792fe0d15a66ffdb96483912e57a11bbaadd5a
+  data.tar.gz: 6212bba57e50c5d6ebaab638ebe89d3690ae9bc9baadcd9581b13953b91f2dbd1a618bf5461a61178030e12abb07f8bdccbed385d5aa673709c8b0707b2b938c

data/CHANGELOG.md

@@ -0,0 +1,25 @@
+# Changelog
+
+## 0.4.0
+
+* Replaced jeweler with a plain Rakefile
+
+## 0.3.1
+
+* Fixes for GCC 4.7
+
+## 0.3.0
+
+* Fixes for Ruby 1.9
+
+## 0.2.0
+
+* Update to SFST 1.3
+
+## 0.1.1
+
+* Overflow and g++ 4.3 fixes
+
+## 0.1.0
+
+* Initial release

data/{README.rdoc → README.md}

@@ -1,18 +1,18 @@
-= ruby-sfst
+# ruby-sfst
 
 A wrapper for the Stuttgart Finite State Transducer Tools (SFST).
 
 The wrapper is based on SFST 1.3. See
-http://www.ims.uni-stuttgart.de/projekte/gramotron/SOFTWARE/SFST.html for
+http://www.cis.uni-muenchen.de/~schmid/tools/SFST/ for
 details on how to obtain SFST and how to write SFST transducers.
 
-== Installation
+## Installation
 
-  gem install ruby-sfst
+    gem install ruby-sfst
 
-== Usage
+## Usage
 
-If you have the following compiled as <tt>test.a</tt>
+If you have the following compiled as `test.a`
 
     ALPHABET = [a-z]
     $test$ = ({bar}:{foo} | {baz}:{foo})
@@ -20,6 +20,7 @@ If you have the following compiled as <tt>test.a</tt>
 
 it can be used like this:
 
+```ruby
     fst = SFST::RegularTransducer.new("test.a")
     fst.analyse('foo')
     # => ['bar', 'baz']
@@ -32,21 +33,24 @@ it can be used like this:
     fst.generate_language { |f| p f }
     # [["b", "f"], ["a", "o"], ["z", "o"]]
     # [["b", "f"], ["a", "o"], ["r", "o"]]
+```
 
 You can also compile an SFST file. This requires the SFST tools to be
 installed and available on the current search path.
 
+```ruby
     SFST::compile("test.fst", "test.a", :compact => true)
+```
 
-Currently, <tt>ruby-sfst</tt> only supports simple compilation, analysis
+Currently, `ruby-sfst` only supports simple compilation, analysis
 and generation using regular and compact transducers. It also only
 supports UTF-8.
 
-== Development
+## Development
 
 The project is hosted on github on http://github.com/mlj/ruby-sfst.
 
-== License
+## License
 
 As it includes the SFST code directly, the wrapper inherits the GPL2
-license of the SFST project.
+license of the SFST project.

data/Rakefile

@@ -1,16 +1,14 @@
-# -*- encoding: utf-8 -*-
-begin
-  require 'jeweler'
+# coding: utf-8
+require 'bundler'
+Bundler::GemHelper.install_tasks
 
-  Jeweler::Tasks.new do |p|
-    p.name = "ruby-sfst"
-    p.summary = "Stuttgart Finite State Transducer Tools interface"
-    p.description = "A wrapper for the Stuttgart Finite State Transducer Tools (SFST)."
-    p.authors = ['Marius L. Jøhndal']
-    p.email = "mariuslj (at) ifi [dot] uio (dot) no"
-    p.homepage = "http://github.com/mlj/ruby-sfst"
-    p.rubyforge_project = "sfst"
+namespace :doc do
+  require 'yard'
+  YARD::Rake::YardocTask.new do |task|
+  task.files   = ['README.md', 'lib/**/*.rb']
+  task.options = [
+    '--output-dir', 'doc/yard',
+    '--markup', 'markdown',
+    ]
   end
-rescue LoadError
-  puts "Jeweler not available. Install it with: sudo gem install jeweler"
 end

data/lib/sfst/version.rb

@@ -0,0 +1,3 @@
+module SFST
+  VERSION = "0.4.0" unless defined?(SFST::VERSION)
+end

data/ruby-sfst.gemspec

@@ -1,73 +1,28 @@
-# Generated by jeweler
-# DO NOT EDIT THIS FILE DIRECTLY
-# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
-# -*- encoding: utf-8 -*-
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'sfst/version'
 
 Gem::Specification.new do |s|
-  s.name = "ruby-sfst"
-  s.version = "0.3.1"
-
-  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
-  s.authors = ["Marius L. J\u{f8}hndal"]
-  s.date = "2013-04-09"
-  s.description = "A wrapper for the Stuttgart Finite State Transducer Tools (SFST)."
-  s.email = "mariuslj (at) ifi [dot] uio (dot) no"
-  s.extensions = ["ext/sfst_machine/extconf.rb"]
-  s.extra_rdoc_files = [
-    "README.rdoc"
-  ]
-  s.files = [
-    "CHANGELOG",
-    "Manifest",
-    "README.rdoc",
-    "Rakefile",
-    "VERSION",
-    "ext/sfst_machine/.gitignore",
-    "ext/sfst_machine/alphabet.C",
-    "ext/sfst_machine/alphabet.h",
-    "ext/sfst_machine/basic.C",
-    "ext/sfst_machine/basic.h",
-    "ext/sfst_machine/compact.C",
-    "ext/sfst_machine/compact.h",
-    "ext/sfst_machine/determinise.C",
-    "ext/sfst_machine/extconf.rb",
-    "ext/sfst_machine/fst-compiler.C",
-    "ext/sfst_machine/fst-compiler.h",
-    "ext/sfst_machine/fst-compiler.yy",
-    "ext/sfst_machine/fst.C",
-    "ext/sfst_machine/fst.h",
-    "ext/sfst_machine/interface.C",
-    "ext/sfst_machine/interface.h",
-    "ext/sfst_machine/make-compact.C",
-    "ext/sfst_machine/make-compact.h",
-    "ext/sfst_machine/mem.h",
-    "ext/sfst_machine/operators.C",
-    "ext/sfst_machine/sfst_machine.cc",
-    "ext/sfst_machine/sgi.h",
-    "ext/sfst_machine/utf8-scanner.C",
-    "ext/sfst_machine/utf8-scanner.ll",
-    "ext/sfst_machine/utf8.C",
-    "ext/sfst_machine/utf8.h",
-    "lib/.gitignore",
-    "lib/sfst.rb",
-    "ruby-sfst.gemspec",
-    "test/.gitignore",
-    "test/test_sfst.fst",
-    "test/test_sfst.rb"
-  ]
+  s.add_development_dependency 'bundler', '~> 1.0'
+  s.authors = ["Marius L. Jøhndal"]
+  s.description = %q{A wrapper for the Stuttgart Finite State Transducer Tools (SFST).}
+  s.summary = %q{Stuttgart Finite State Transducer Tools interface}
+  s.email = ['mariuslj (at) ifi [dot] uio (dot) no']
+  s.files = %w(CHANGELOG.md README.md Rakefile ruby-sfst.gemspec)
+  s.files += Dir.glob("ext/**/*.C")
+  s.files += Dir.glob("ext/**/*.h")
+  s.files += Dir.glob("ext/**/*.rb")
+  s.files += Dir.glob("ext/**/*.cc")
+  s.files += Dir.glob("lib/**/*.rb")
+  s.files += Dir.glob("test/*.fst")
+  s.files += Dir.glob("test/*.rb")
   s.homepage = "http://github.com/mlj/ruby-sfst"
+  s.licenses = ['GPL2']
+  s.name = "ruby-sfst"
   s.require_paths = ["lib"]
-  s.rubyforge_project = "sfst"
-  s.rubygems_version = "1.8.23"
-  s.summary = "Stuttgart Finite State Transducer Tools interface"
-
-  if s.respond_to? :specification_version then
-    s.specification_version = 3
-
-    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-    else
-    end
-  else
-  end
+  s.required_rubygems_version = '>= 1.3.5'
+  s.extensions = ["ext/sfst_machine/extconf.rb"]
+  s.test_files += Dir.glob("test/*.rb")
+  s.version = SFST::VERSION
 end
-

metadata

@@ -1,83 +1,92 @@
 --- !ruby/object:Gem::Specification
 name: ruby-sfst
 version: !ruby/object:Gem::Version
-  version: 0.3.1
-  prerelease: 
+  version: 0.4.0
 platform: ruby
 authors:
 - Marius L. Jøhndal
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-04-09 00:00:00.000000000 Z
-dependencies: []
+date: 2013-09-10 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
 description: A wrapper for the Stuttgart Finite State Transducer Tools (SFST).
-email: mariuslj (at) ifi [dot] uio (dot) no
+email:
+- mariuslj (at) ifi [dot] uio (dot) no
 executables: []
 extensions:
 - ext/sfst_machine/extconf.rb
-extra_rdoc_files:
-- README.rdoc
+extra_rdoc_files: []
 files:
-- CHANGELOG
-- Manifest
-- README.rdoc
+- CHANGELOG.md
+- README.md
 - Rakefile
-- VERSION
-- ext/sfst_machine/.gitignore
-- ext/sfst_machine/alphabet.C
-- ext/sfst_machine/alphabet.h
-- ext/sfst_machine/basic.C
-- ext/sfst_machine/basic.h
+- ruby-sfst.gemspec
 - ext/sfst_machine/compact.C
-- ext/sfst_machine/compact.h
+- ext/sfst_machine/fst.C
+- ext/sfst_machine/interface.C
+- ext/sfst_machine/make-compact.C
 - ext/sfst_machine/determinise.C
-- ext/sfst_machine/extconf.rb
 - ext/sfst_machine/fst-compiler.C
-- ext/sfst_machine/fst-compiler.h
-- ext/sfst_machine/fst-compiler.yy
-- ext/sfst_machine/fst.C
+- ext/sfst_machine/utf8.C
+- ext/sfst_machine/utf8-scanner.C
+- ext/sfst_machine/alphabet.C
+- ext/sfst_machine/basic.C
+- ext/sfst_machine/operators.C
 - ext/sfst_machine/fst.h
-- ext/sfst_machine/interface.C
+- ext/sfst_machine/fst-compiler.h
+- ext/sfst_machine/mem.h
+- ext/sfst_machine/basic.h
+- ext/sfst_machine/alphabet.h
 - ext/sfst_machine/interface.h
-- ext/sfst_machine/make-compact.C
 - ext/sfst_machine/make-compact.h
-- ext/sfst_machine/mem.h
-- ext/sfst_machine/operators.C
-- ext/sfst_machine/sfst_machine.cc
+- ext/sfst_machine/compact.h
 - ext/sfst_machine/sgi.h
-- ext/sfst_machine/utf8-scanner.C
-- ext/sfst_machine/utf8-scanner.ll
-- ext/sfst_machine/utf8.C
 - ext/sfst_machine/utf8.h
-- lib/.gitignore
+- ext/sfst_machine/extconf.rb
+- ext/sfst_machine/sfst_machine.cc
 - lib/sfst.rb
-- ruby-sfst.gemspec
-- test/.gitignore
+- lib/sfst/version.rb
 - test/test_sfst.fst
 - test/test_sfst.rb
 homepage: http://github.com/mlj/ruby-sfst
-licenses: []
+licenses:
+- GPL2
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.5
 requirements: []
-rubyforge_project: sfst
-rubygems_version: 1.8.23
+rubyforge_project: 
+rubygems_version: 2.0.3
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Stuttgart Finite State Transducer Tools interface
-test_files: []
+test_files:
+- test/test_sfst.rb
+has_rdoc: 

data/CHANGELOG

@@ -1,5 +0,0 @@
-v0.3.1. fixes for GCC 4.7.
-v0.3.0. fixes for Ruby 1.9
-v0.2.0. update to SFST 1.3
-v0.1.1. overflow and g++ 4.3 fixes
-v0.1.0. initial release

data/Manifest

@@ -1,32 +0,0 @@
-README.rdoc
-Rakefile
-Manifest
-test/test_sfst.rb
-test/test_sfst.fst
-CHANGELOG
-ext/sfst_machine/fst-compiler.h
-ext/sfst_machine/utf8.C
-ext/sfst_machine/operators.C
-ext/sfst_machine/utf8-scanner.ll
-ext/sfst_machine/determinise.C
-ext/sfst_machine/interface.C
-ext/sfst_machine/compact.h
-ext/sfst_machine/sgi.h
-ext/sfst_machine/basic.h
-ext/sfst_machine/fst.h
-ext/sfst_machine/make-compact.h
-ext/sfst_machine/fst-compiler.yy
-ext/sfst_machine/mem.h
-ext/sfst_machine/compact.C
-ext/sfst_machine/basic.C
-ext/sfst_machine/interface.h
-ext/sfst_machine/sfst_machine.cc
-ext/sfst_machine/extconf.rb
-ext/sfst_machine/alphabet.C
-ext/sfst_machine/fst.C
-ext/sfst_machine/alphabet.h
-ext/sfst_machine/make-compact.C
-ext/sfst_machine/fst-compiler.C
-ext/sfst_machine/utf8.h
-ext/sfst_machine/utf8-scanner.C
-lib/sfst.rb

data/VERSION

@@ -1 +0,0 @@
-0.3.1

data/ext/sfst_machine/.gitignore

@@ -1,3 +0,0 @@
-*.so
-*.o
-Makefile

data/ext/sfst_machine/fst-compiler.yy

@@ -1,213 +0,0 @@
-%{
-/*******************************************************************/
-/*                                                                 */
-/*  FILE     fst-compiler.yy                                       */
-/*  MODULE   fst-compiler                                          */
-/*  PROGRAM  SFST                                                  */
-/*  AUTHOR   Helmut Schmid, IMS, University of Stuttgart           */
-/*                                                                 */
-/*******************************************************************/
-
-#include <stdio.h>
-
-#include "make-compact.h"
-#include "interface.h"
-
-using std::cerr;
-
-extern int  yylineno;
-extern char *yytext;
-
-void yyerror(char *text);
-void warn(char *text);
-void warn2(char *text, char *text2);
-int yylex( void );
-int yyparse( void );
-
-static int Switch=0;
-Transducer *Result;
-%}
-
-%union {
-  int        number;
-  Twol_Type  type;
-  Repl_Type  rtype;
-  char       *name;
-  char       *value;
-  unsigned char uchar;
-  unsigned int  longchar;
-  Character  character;
-  Transducer  *expression;
-  Range      *range;
-  Ranges     *ranges;
-  Contexts   *contexts;
-}
-
-%token <number> NEWLINE ALPHA COMPOSE PRINT POS INSERT SWITCH
-%token <type>   ARROW
-%token <rtype>  REPLACE
-%token <name>   SYMBOL VAR SVAR RVAR RSVAR
-%token <value>  STRING STRING2 UTF8CHAR
-%token <uchar>  CHARACTER
-
-%type  <uchar>      SCHAR
-%type  <longchar>   LCHAR
-%type  <character>  CODE
-%type  <expression> RE
-%type  <range>      RANGE VALUE VALUES
-%type  <ranges>     RANGES
-%type  <contexts>   CONTEXT CONTEXT2 CONTEXTS CONTEXTS2
-
-%left PRINT INSERT
-%left ARROW REPLACE
-%left COMPOSE
-%left '|'
-%left '-'
-%left '&'
-%left SEQ
-%left '!' '^' '_'
-%left '*' '+'
-%%
-
-ALL:        ASSIGNMENTS RE NEWLINES { Result=result($2, Switch); }
-          ;
-
-ASSIGNMENTS: ASSIGNMENTS ASSIGNMENT {}
-          | ASSIGNMENTS NEWLINE     {}
-          | /* nothing */           {}
-          ;
-
-ASSIGNMENT: VAR '=' RE              { if (def_var($1,$3)) warn2("assignment of empty transducer to",$1); }
-          | RVAR '=' RE             { if (def_rvar($1,$3)) warn2("assignment of empty transducer to",$1); }
-          | SVAR '=' VALUES         { if (def_svar($1,$3)) warn2("assignment of empty symbol range to",$1); }
-          | RSVAR '=' VALUES        { if (def_svar($1,$3)) warn2("assignment of empty symbol range to",$1); }
-          | RE PRINT STRING         { write_to_file($1, $3); }
-          | ALPHA RE                { def_alphabet($2); }
-          ;
-
-RE:         RE ARROW CONTEXTS2      { $$ = restriction($1,$2,$3,0); }
-	  | RE '^' ARROW CONTEXTS2  { $$ = restriction($1,$3,$4,1); }
-	  | RE '_' ARROW CONTEXTS2  { $$ = restriction($1,$3,$4,-1); }
-          | RE REPLACE CONTEXT2     { $$ = replace_in_context(minimise(explode($1)),$2,$3,false); }
-          | RE REPLACE '?' CONTEXT2 { $$ = replace_in_context(minimise(explode($1)),$2,$4,true);}
-          | RE REPLACE '(' ')'      { $$ = replace(minimise(explode($1)), $2, false); }
-          | RE REPLACE '?' '(' ')'  { $$ = replace(minimise(explode($1)), $2, true); }
-          | RE RANGE ARROW RANGE RE { $$ = make_rule($1,$2,$3,$4,$5); }
-          | RE RANGE ARROW RANGE    { $$ = make_rule($1,$2,$3,$4,NULL); }
-          | RANGE ARROW RANGE RE    { $$ = make_rule(NULL,$1,$2,$3,$4); }
-          | RANGE ARROW RANGE       { $$ = make_rule(NULL,$1,$2,$3,NULL); }
-          | RE COMPOSE RE    { $$ = composition($1, $3); }
-          | '{' RANGES '}' ':' '{' RANGES '}' { $$ = make_mapping($2,$6); }
-          | RANGE ':' '{' RANGES '}' { $$ = make_mapping(add_range($1,NULL),$4); }
-          | '{' RANGES '}' ':' RANGE { $$ = make_mapping($2,add_range($5,NULL)); }
-          | RE INSERT CODE ':' CODE  { $$ = freely_insert($1, $3, $5); }
-          | RE INSERT CODE           { $$ = freely_insert($1, $3, $3); }
-          | RANGE ':' RANGE  { $$ = new_transducer($1,$3); }
-          | RANGE            { $$ = new_transducer($1,$1); }
-          | VAR              { $$ = var_value($1); }
-          | RVAR             { $$ = rvar_value($1); }
-          | RE '*'           { $$ = repetition($1); }
-          | RE '+'           { $$ = repetition2($1); }
-          | RE '?'           { $$ = optional($1); }
-          | RE RE %prec SEQ  { $$ = catenate($1, $2); }
-          | '!' RE           { $$ = negation($2); }
-          | SWITCH RE        { $$ = switch_levels($2); }
-          | '^' RE           { $$ = upper_level($2); }
-          | '_' RE           { $$ = lower_level($2); }
-          | RE '&' RE        { $$ = conjunction($1, $3); }
-          | RE '-' RE        { $$ = subtraction($1, $3); }
-          | RE '|' RE        { $$ = disjunction($1, $3); }
-          | '(' RE ')'       { $$ = $2; }
-          | STRING           { $$ = read_words($1); }
-          | STRING2          { $$ = read_transducer($1); }
-          ;
-
-RANGES:     RANGE RANGES     { $$ = add_range($1,$2); }
-          |                  { $$ = NULL; }
-          ;
-
-RANGE:      '[' VALUES ']'   { $$=$2; }
-          | '[' '^' VALUES ']' { $$=complement_range($3); }
-          | '[' RSVAR ']'    { $$=rsvar_value($2); }
-          | '.'              { $$=NULL; }
-          | CODE             { $$=add_value($1,NULL); }
-          ;
-
-CONTEXTS2:  CONTEXTS               { $$ = $1; }
-          | '(' CONTEXTS ')'       { $$ = $2; }
-          ;
-
-CONTEXTS:   CONTEXT ',' CONTEXTS   { $$ = add_context($1,$3); }
-          | CONTEXT                { $$ = $1; }
-          ;
-
-CONTEXT2:   CONTEXT                { $$ = $1; }
-          | '(' CONTEXT ')'        { $$ = $2; }
-          ;
-
-CONTEXT :   RE POS RE              { $$ = make_context($1, $3); }
-          |    POS RE              { $$ = make_context(NULL, $2); }
-          | RE POS                 { $$ = make_context($1, NULL); }
-          ;
-
-VALUES:     VALUE VALUES           { $$=append_values($1,$2); }
-          | VALUE                  { $$ = $1; }
-          ;
-
-VALUE:      LCHAR '-' LCHAR	   { $$=add_values($1,$3,NULL); }
-          | SVAR                   { $$=svar_value($1); }
-          | LCHAR  	           { $$=add_value(character_code($1),NULL); }
-          | CODE		   { $$=add_value($1,NULL); }
-	  | SCHAR		   { $$=add_value($1,NULL); }
-          ;
-
-LCHAR:      CHARACTER	{ $$=$1; }
-          | UTF8CHAR	{ $$=utf8toint($1); free($1); }
-	  | SCHAR       { $$=$1; }
-          ;
-
-CODE:       CHARACTER	{ $$=character_code($1); }
-          | UTF8CHAR	{ $$=symbol_code($1); }
-          | SYMBOL	{ $$=symbol_code($1); }
-          ;
-
-SCHAR:      '.'		{ $$=character_code('.'); }
-          | '!'		{ $$=character_code('!'); }
-          | '?'		{ $$=character_code('?'); }
-          | '{'		{ $$=character_code('{'); }
-          | '}'		{ $$=character_code('}'); }
-          | ')'		{ $$=character_code(')'); }
-          | '('		{ $$=character_code('('); }
-          | '&'		{ $$=character_code('&'); }
-          | '|'		{ $$=character_code('|'); }
-          | '*'		{ $$=character_code('*'); }
-          | '+'		{ $$=character_code('+'); }
-          | ':'		{ $$=character_code(':'); }
-          | ','		{ $$=character_code(','); }
-          | '='		{ $$=character_code('='); }
-          | '_'		{ $$=character_code('_'); }
-          | '^'		{ $$=character_code('^'); }
-          | '-'		{ $$=character_code('-'); }
-          ;
-
-NEWLINES:   NEWLINE NEWLINES     {}
-          | /* nothing */        {}
-          ;
-
-%%
-
-extern FILE  *yyin;
-
-/*******************************************************************/
-/*                                                                 */
-/*  yyerror                                                        */
-/*                                                                 */
-/*******************************************************************/
-
-void yyerror(char *text)
-
-{
-  cerr << "\n" << FileName << ":" << yylineno << ": " << text << " at: ";
-  cerr << yytext << "\naborted.\n";
-  exit(1);
-}

data/ext/sfst_machine/utf8-scanner.ll

@@ -1,175 +0,0 @@
-%option 8Bit batch yylineno noyywrap
-
-/* the "incl" state is used to pick up the name of an include file */
-%x incl
-
-%{
-/*******************************************************************/
-/*                                                                 */
-/*  FILE     scanner.ll                                            */
-/*  MODULE   scanner                                               */
-/*  PROGRAM  SFST                                                  */
-/*  AUTHOR   Helmut Schmid, IMS, University of Stuttgart           */
-/*                                                                 */
-/*******************************************************************/
-
-#include <string.h>
-
-#include "interface.h"
-#include "fst-compiler.h"
-
-#define MAX_INCLUDE_DEPTH 10
-  
-int Include_Stack_Ptr = 0;
-YY_BUFFER_STATE Include_Stack[MAX_INCLUDE_DEPTH];
-char *Name_Stack[MAX_INCLUDE_DEPTH];
-int  Lineno_Stack[MAX_INCLUDE_DEPTH];
-
-char *FileName;
-
-bool UTF8=true;
-
-static char *unquote(char *string, bool del_quote=true) {
-  char *s=string, *result=string;
-  if (del_quote)
-    string++;
-
-  while (*string) {
-    if (*string == '\\')
-      string++;
-    *(s++) = *(string++);
-  }
-
-  if (del_quote)
-    s--;
-  *s = '\0';
-
-  return fst_strdup(result);
-}
-
-static void print_lineno() {
-  if (!Verbose)
-    return;
-  fputc('\r',stderr);
-  for( int i=0; i<Include_Stack_Ptr; i++ )
-    fputs("  ", stderr);
-  fprintf(stderr,"%s: %d", FileName, yylineno);
-}
-
-extern void yyerror(char *text);
-
-%}
-
-CC	[\x80-\xbf]
-C1	[A-Za-z0-9._/\-]
-C2	[A-Za-z0-9._/\-&()+,=?\^|~]
-C3	[A-Za-z0-9._/\-&()+,=?\^|~#<>]
-C4	[A-Za-z0-9._/\-&()+,=?\^|~$<>]
-C5	[\!-;\?-\[\]-\~=]
-FN	[A-Za-z0-9._/\-*+]
-
-%%
-
-#include           BEGIN(incl);
-<incl>[ \t]*       /* eat the whitespace */
-<incl>{FN}+        { error2("Missing quotes",yytext); }
-<incl>\"{FN}+\"    { /* got the include file name */
-                     FILE *file;
-                     char *name=unquote(yytext);
-                     if ( Include_Stack_Ptr >= MAX_INCLUDE_DEPTH ) {
-		       fprintf( stderr, "Includes nested too deeply" );
-		       exit( 1 );
-		     }
-		     if (Verbose) fputc('\n', stderr);
-		     file = fopen( name, "rt" );
-		     if (!file)
-                       error2("Can't open include file", name);
-                     else {
-                       Name_Stack[Include_Stack_Ptr] = FileName;
-                       FileName = name;
-                       Lineno_Stack[Include_Stack_Ptr] = yylineno;
-		       yylineno = 1;
-		       Include_Stack[Include_Stack_Ptr++]=YY_CURRENT_BUFFER;
-		       yy_switch_to_buffer(yy_create_buffer(yyin, YY_BUF_SIZE));
-                       yyin = file;
-		       print_lineno();
-		       BEGIN(INITIAL);
-                     }
-                  }
-<<EOF>>           {
-                     if (Verbose)
-		       fputc('\n', stderr);
-                     if ( --Include_Stack_Ptr < 0 )
-		       yyterminate();
-		     else {
-                       free(FileName);
-                       FileName = Name_Stack[Include_Stack_Ptr];
-                       yylineno = Lineno_Stack[Include_Stack_Ptr];
-		       yy_delete_buffer( YY_CURRENT_BUFFER );
-		       yy_switch_to_buffer(Include_Stack[Include_Stack_Ptr]);
-                     }
-                  }
-
-
-^[ \t]*\%.*\r?\n  { print_lineno();  /* ignore comments */ }
-
-\%.*\\[ \t]*\r?\n { print_lineno();  /* ignore comments */ }
-
-\%.*              { /* ignore comments */ }
-
-
-^[ \t]*ALPHABET[ \t]*= { return ALPHA; }
-
-\|\|              { return COMPOSE; }
-"<=>"             { yylval.type = twol_both; return ARROW; }
-"=>"              { yylval.type = twol_right;return ARROW; }
-"<="              { yylval.type = twol_left; return ARROW; }
-"^->"             { yylval.rtype = repl_up;   return REPLACE; }
-"_->"             { yylval.rtype = repl_down; return REPLACE; }
-"/->"             { yylval.rtype = repl_right;return REPLACE; }
-"\\->"            { yylval.rtype = repl_left; return REPLACE; }
-">>"              { return PRINT; }
-"<<"              { return INSERT; }
-"__"              { return POS; }
-"^_"              { return SWITCH; }
-
-[.,{}\[\]()&!?|*+:=_\^\-] { return yytext[0]; }
-
-\$=({C3}|(\\.))+\$ { yylval.name = fst_strdup(yytext); return RVAR; }
-
-\$({C3}|(\\.))+\$ { yylval.name = fst_strdup(yytext); return VAR; }
-
-#=({C4}|(\\.))+# { yylval.name = fst_strdup(yytext); return RSVAR; }
-
-#({C4}|(\\.))+# { yylval.name = fst_strdup(yytext); return SVAR; }
-
-\<({C5}|\\.)*\>   { yylval.name = unquote(yytext,false); return SYMBOL; }
-
-\"<{FN}+>\" { 
-                    yylval.value = unquote(yytext)+1;
-		    yylval.value[strlen(yylval.value)-1] = 0;
-                    return STRING2;
-                  }
-
-\"{FN}+\" { 
-                    yylval.value = unquote(yytext);
-                    return STRING;
-                  }
-
-[ \t]             { /* ignored */ }
-\\[ \t]*([ \t]\%.*)?\r?\n { print_lineno(); /* ignored */ }
-\r?\n             { print_lineno(); return NEWLINE; }
-
-\\[0-9]+          { long l=atol(yytext+1); 
-		    if (l <= 1114112) { yylval.uchar=l; return CHARACTER; }
-		    yyerror("invalid expression");
-                  }
-
-
-\\.                { yylval.value=fst_strdup(yytext+1); return UTF8CHAR; }
-[\x00-\x7f]        { yylval.value=fst_strdup(yytext); return UTF8CHAR; }
-[\xc0-\xdf]{CC}    { yylval.value=fst_strdup(yytext); return UTF8CHAR; }
-[\xe0-\xef]{CC}{2} { yylval.value=fst_strdup(yytext); return UTF8CHAR; }
-[\xf0-\xff]{CC}{3} { yylval.value=fst_strdup(yytext); return UTF8CHAR; }
-
-%%

data/lib/.gitignore

@@ -1 +0,0 @@
-*.so

data/test/.gitignore

@@ -1 +0,0 @@
-*.a