ruby-sfst 0.1.0

data/ext/sfst_machine/fst-compiler.h

@@ -0,0 +1,113 @@
+/* A Bison parser, made by GNU Bison 2.3.  */
+
+/* Skeleton interface for Bison's Yacc-like parsers in C
+
+   Copyright (C) 1984, 1989, 1990, 2000, 2001, 2002, 2003, 2004, 2005, 2006
+   Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor,
+   Boston, MA 02110-1301, USA.  */
+
+/* As a special exception, you may create a larger work that contains
+   part or all of the Bison parser skeleton and distribute that work
+   under terms of your choice, so long as that work isn't itself a
+   parser generator using the skeleton or a modified version thereof
+   as a parser skeleton.  Alternatively, if you modify or redistribute
+   the parser skeleton itself, you may (at your option) remove this
+   special exception, which will cause the skeleton and the resulting
+   Bison output files to be licensed under the GNU General Public
+   License without this special exception.
+
+   This special exception was added by the Free Software Foundation in
+   version 2.2 of Bison.  */
+
+/* Tokens.  */
+#ifndef YYTOKENTYPE
+# define YYTOKENTYPE
+   /* Put the tokens into the symbol table, so that GDB and other debuggers
+      know about them.  */
+   enum yytokentype {
+     NEWLINE = 258,
+     ALPHA = 259,
+     COMPOSE = 260,
+     PRINT = 261,
+     POS = 262,
+     INSERT = 263,
+     REV = 264,
+     ARROW = 265,
+     REPLACE = 266,
+     SYMBOL = 267,
+     VAR = 268,
+     SVAR = 269,
+     RVAR = 270,
+     RSVAR = 271,
+     STRING = 272,
+     STRING2 = 273,
+     UTF8CHAR = 274,
+     CHARACTER = 275,
+     SEQ = 276
+   };
+#endif
+/* Tokens.  */
+#define NEWLINE 258
+#define ALPHA 259
+#define COMPOSE 260
+#define PRINT 261
+#define POS 262
+#define INSERT 263
+#define REV 264
+#define ARROW 265
+#define REPLACE 266
+#define SYMBOL 267
+#define VAR 268
+#define SVAR 269
+#define RVAR 270
+#define RSVAR 271
+#define STRING 272
+#define STRING2 273
+#define UTF8CHAR 274
+#define CHARACTER 275
+#define SEQ 276
+
+
+
+
+#if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED
+typedef union YYSTYPE
+#line 31 "fst-compiler.yy"
+{
+  int        number;
+  Twol_Type  type;
+  Repl_Type  rtype;
+  char       *name;
+  char       *value;
+  unsigned char uchar;
+  unsigned int  longchar;
+  Character  character;
+  Transducer  *expression;
+  Range      *range;
+  Ranges     *ranges;
+  Contexts   *contexts;
+}
+/* Line 1489 of yacc.c.  */
+#line 106 "fst-compiler.H"
+	YYSTYPE;
+# define yystype YYSTYPE /* obsolescent; will be withdrawn */
+# define YYSTYPE_IS_DECLARED 1
+# define YYSTYPE_IS_TRIVIAL 1
+#endif
+
+extern YYSTYPE yylval;
+

data/ext/sfst_machine/fst-compiler.yy

@@ -0,0 +1,213 @@
+%{
+/*******************************************************************/
+/*                                                                 */
+/*  FILE     fst-compiler.yy                                       */
+/*  MODULE   fst-compiler                                          */
+/*  PROGRAM  SFST                                                  */
+/*  AUTHOR   Helmut Schmid, IMS, University of Stuttgart           */
+/*                                                                 */
+/*******************************************************************/
+
+#include <stdio.h>
+
+#include "make-compact.h"
+#include "interface.h"
+
+using std::cerr;
+
+extern int  yylineno;
+extern char *yytext;
+
+void yyerror(char *text);
+void warn(char *text);
+void warn2(char *text, char *text2);
+int yylex( void );
+int yyparse( void );
+
+static int Switch=0;
+Transducer *Result;
+%}
+
+%union {
+  int        number;
+  Twol_Type  type;
+  Repl_Type  rtype;
+  char       *name;
+  char       *value;
+  unsigned char uchar;
+  unsigned int  longchar;
+  Character  character;
+  Transducer  *expression;
+  Range      *range;
+  Ranges     *ranges;
+  Contexts   *contexts;
+}
+
+%token <number> NEWLINE ALPHA COMPOSE PRINT POS INSERT REV
+%token <type>   ARROW
+%token <rtype>  REPLACE
+%token <name>   SYMBOL VAR SVAR RVAR RSVAR
+%token <value>  STRING STRING2 UTF8CHAR
+%token <uchar>  CHARACTER
+
+%type  <uchar>      SCHAR
+%type  <longchar>   LCHAR
+%type  <character>  CODE
+%type  <expression> RE
+%type  <range>      RANGE VALUE VALUES
+%type  <ranges>     RANGES
+%type  <contexts>   CONTEXT CONTEXT2 CONTEXTS CONTEXTS2
+
+%left PRINT INSERT
+%left ARROW REPLACE
+%left COMPOSE
+%left '|'
+%left '-'
+%left '&'
+%left SEQ
+%left '!' '^' '_'
+%left '*' '+'
+%%
+
+ALL:        ASSIGNMENTS RE NEWLINES { Result=result($2, Switch); }
+          ;
+
+ASSIGNMENTS: ASSIGNMENTS ASSIGNMENT {}
+          | ASSIGNMENTS NEWLINE     {}
+          | /* nothing */           {}
+          ;
+
+ASSIGNMENT: VAR '=' RE              { if (def_var($1,$3)) warn2("assignment of empty transducer to",$1); }
+          | RVAR '=' RE             { if (def_rvar($1,$3)) warn2("assignment of empty transducer to",$1); }
+          | SVAR '=' VALUES         { if (def_svar($1,$3)) warn2("assignment of empty symbol range to",$1); }
+          | RSVAR '=' VALUES        { if (def_svar($1,$3)) warn2("assignment of empty symbol range to",$1); }
+          | RE PRINT STRING         { write_to_file($1, $3); }
+          | ALPHA RE                { def_alphabet($2); }
+          ;
+
+RE:         RE ARROW CONTEXTS2      { $$ = restriction($1,$2,$3,0); }
+	  | RE '^' ARROW CONTEXTS2  { $$ = restriction($1,$3,$4,1); }
+	  | RE '_' ARROW CONTEXTS2  { $$ = restriction($1,$3,$4,-1); }
+          | RE REPLACE CONTEXT2     { $$ = replace_in_context(minimise(explode($1)),$2,$3,false); }
+          | RE REPLACE '?' CONTEXT2 { $$ = replace_in_context(minimise(explode($1)),$2,$4,true);}
+          | RE REPLACE '(' ')'      { $$ = replace(minimise(explode($1)), $2, false); }
+          | RE REPLACE '?' '(' ')'  { $$ = replace(minimise(explode($1)), $2, true); }
+          | RE RANGE ARROW RANGE RE { $$ = make_rule($1,$2,$3,$4,$5); }
+          | RE RANGE ARROW RANGE    { $$ = make_rule($1,$2,$3,$4,NULL); }
+          | RANGE ARROW RANGE RE    { $$ = make_rule(NULL,$1,$2,$3,$4); }
+          | RANGE ARROW RANGE       { $$ = make_rule(NULL,$1,$2,$3,NULL); }
+          | RE COMPOSE RE    { $$ = composition($1, $3); }
+          | '{' RANGES '}' ':' '{' RANGES '}' { $$ = make_mapping($2,$6); }
+          | RANGE ':' '{' RANGES '}' { $$ = make_mapping(add_range($1,NULL),$4); }
+          | '{' RANGES '}' ':' RANGE { $$ = make_mapping($2,add_range($5,NULL)); }
+          | RE INSERT CODE ':' CODE  { $$ = freely_insert($1, $3, $5); }
+          | RE INSERT CODE           { $$ = freely_insert($1, $3, $3); }
+          | RANGE ':' RANGE  { $$ = new_transducer($1,$3); }
+          | RANGE            { $$ = new_transducer($1,$1); }
+          | VAR              { $$ = var_value($1); }
+          | RVAR             { $$ = rvar_value($1); }
+          | RE '*'           { $$ = repetition($1); }
+          | RE '+'           { $$ = repetition2($1); }
+          | RE '?'           { $$ = optional($1); }
+          | RE RE %prec SEQ  { $$ = catenate($1, $2); }
+          | '!' RE           { $$ = negation($2); }
+          | REV RE           { $$ = switch_levels($2); }
+          | '^' RE           { $$ = upper_level($2); }
+          | '_' RE           { $$ = lower_level($2); }
+          | RE '&' RE        { $$ = conjunction($1, $3); }
+          | RE '-' RE        { $$ = subtraction($1, $3); }
+          | RE '|' RE        { $$ = disjunction($1, $3); }
+          | '(' RE ')'       { $$ = $2; }
+          | STRING           { $$ = read_words($1); }
+          | STRING2          { $$ = read_transducer($1); }
+          ;
+
+RANGES:     RANGE RANGES     { $$ = add_range($1,$2); }
+          |                  { $$ = NULL; }
+          ;
+
+RANGE:      '[' VALUES ']'   { $$=$2; }
+          | '[' '^' VALUES ']' { $$=complement_range($3); }
+          | '[' RSVAR ']'    { $$=rsvar_value($2); }
+          | '.'              { $$=NULL; }
+          | CODE             { $$=add_value($1,NULL); }
+          ;
+
+CONTEXTS2:  CONTEXTS               { $$ = $1; }
+          | '(' CONTEXTS ')'       { $$ = $2; }
+          ;
+
+CONTEXTS:   CONTEXT ',' CONTEXTS   { $$ = add_context($1,$3); }
+          | CONTEXT                { $$ = $1; }
+          ;
+
+CONTEXT2:   CONTEXT                { $$ = $1; }
+          | '(' CONTEXT ')'        { $$ = $2; }
+          ;
+
+CONTEXT :   RE POS RE              { $$ = make_context($1, $3); }
+          |    POS RE              { $$ = make_context(NULL, $2); }
+          | RE POS                 { $$ = make_context($1, NULL); }
+          ;
+
+VALUES:     VALUE VALUES           { $$=append_values($1,$2); }
+          | VALUE                  { $$ = $1; }
+          ;
+
+VALUE:      LCHAR '-' LCHAR	   { $$=add_values($1,$3,NULL); }
+          | SVAR                   { $$=svar_value($1); }
+          | LCHAR  	           { $$=add_value(character_code($1),NULL); }
+          | CODE		   { $$=add_value($1,NULL); }
+	  | SCHAR		   { $$=add_value($1,NULL); }
+          ;
+
+LCHAR:      CHARACTER	{ $$=$1; }
+          | UTF8CHAR	{ $$=utf8toint($1); }
+	  | SCHAR       { $$=$1; }
+          ;
+
+CODE:       CHARACTER	{ $$=character_code($1); }
+          | UTF8CHAR	{ $$=symbol_code($1); }
+          | SYMBOL	{ $$=symbol_code($1); }
+          ;
+
+SCHAR:      '.'		{ $$=character_code('.'); }
+          | '!'		{ $$=character_code('!'); }
+          | '?'		{ $$=character_code('?'); }
+          | '{'		{ $$=character_code('{'); }
+          | '}'		{ $$=character_code('}'); }
+          | ')'		{ $$=character_code(')'); }
+          | '('		{ $$=character_code('('); }
+          | '&'		{ $$=character_code('&'); }
+          | '|'		{ $$=character_code('|'); }
+          | '*'		{ $$=character_code('*'); }
+          | '+'		{ $$=character_code('+'); }
+          | ':'		{ $$=character_code(':'); }
+          | ','		{ $$=character_code(','); }
+          | '='		{ $$=character_code('='); }
+          | '_'		{ $$=character_code('_'); }
+          | '^'		{ $$=character_code('^'); }
+          | '-'		{ $$=character_code('-'); }
+          ;
+
+NEWLINES:   NEWLINE NEWLINES     {}
+          | /* nothing */        {}
+          ;
+
+%%
+
+extern FILE  *yyin;
+
+/*******************************************************************/
+/*                                                                 */
+/*  yyerror                                                        */
+/*                                                                 */
+/*******************************************************************/
+
+void yyerror(char *text)
+
+{
+  cerr << "\n" << FileName << ":" << yylineno << ": " << text << " at: ";
+  cerr << yytext << "\naborted.\n";
+  exit(1);
+}