langscan 1.2

data/ext/langscan/sh/sh/sh.h

@@ -0,0 +1,61 @@
+#ifndef LANGSCAN_SH_H
+#define LANGSCAN_SH_H
+
+#define LANGSCAN_SH_TOKEN_LIST \
+     LANGSCAN_SH_TOKEN(string) \
+     LANGSCAN_SH_TOKEN(fundef) \
+     LANGSCAN_SH_TOKEN(ident) \
+     LANGSCAN_SH_TOKEN(punct) \
+     LANGSCAN_SH_TOKEN(comment) \
+     LANGSCAN_SH_TOKEN(space) \
+     LANGSCAN_SH_TOKEN(heredoc_beg) \
+     LANGSCAN_SH_TOKEN(heredoc_end)
+
+typedef enum {
+  langscan_sh_eof = 0,
+#define LANGSCAN_SH_TOKEN(name) langscan_sh_##name,
+  LANGSCAN_SH_TOKEN_LIST
+#undef LANGSCAN_SH_TOKEN
+} langscan_sh_token_t;
+
+typedef struct {
+  int beg_lineno;
+  int beg_columnno;
+  int beg_byteno;
+  int end_lineno;
+  int end_columnno;
+  int end_byteno;
+  int eof;
+  char *text;
+  int leng;
+  size_t (*user_read)(void **user_data_p, char *buf, size_t maxlen);
+  void *user_data;
+} langscan_sh_lex_extra_t;
+
+typedef struct langscan_sh_tokenizer_tag {
+  langscan_sh_lex_extra_t *extra;
+  void *scanner;
+} langscan_sh_tokenizer_t;
+
+typedef size_t (*user_read_t)(void **user_data_p, char *buf, size_t maxlen);
+
+langscan_sh_tokenizer_t *langscan_sh_make_tokenizer(user_read_t user_read, void *user_data);
+langscan_sh_token_t langscan_sh_get_token(langscan_sh_tokenizer_t *tokenizer);
+void langscan_sh_free_tokenizer(langscan_sh_tokenizer_t *tokenizer);
+
+user_read_t langscan_sh_tokenizer_get_user_read(langscan_sh_tokenizer_t *tokenizer);
+void *langscan_sh_tokenizer_get_user_data(langscan_sh_tokenizer_t *tokenizer);
+
+const char *langscan_sh_token_name(langscan_sh_token_t token);
+#define langscan_sh_curtoken_beg_lineno(tokenizer) ((tokenizer)->extra->beg_lineno)
+#define langscan_sh_curtoken_beg_columnno(tokenizer) ((tokenizer)->extra->beg_columnno)
+#define langscan_sh_curtoken_beg_byteno(tokenizer) ((tokenizer)->extra->beg_byteno)
+#define langscan_sh_curtoken_end_lineno(tokenizer) ((tokenizer)->extra->end_lineno)
+#define langscan_sh_curtoken_end_columnno(tokenizer) ((tokenizer)->extra->end_columnno)
+#define langscan_sh_curtoken_end_byteno(tokenizer) ((tokenizer)->extra->end_byteno)
+#define langscan_sh_curtoken_text(tokenizer) ((tokenizer)->extra->text)
+#define langscan_sh_curtoken_leng(tokenizer) ((tokenizer)->extra->leng)
+
+void langscan_sh_extract_functions(langscan_sh_tokenizer_t *);
+
+#endif

data/ext/langscan/sh/sh/shtok.c

@@ -0,0 +1,2470 @@
+
+#line 3 "<stdout>"
+
+#define  YY_INT_ALIGNED short int
+
+/* A lexical scanner generated by flex */
+
+#define FLEX_SCANNER
+#define YY_FLEX_MAJOR_VERSION 2
+#define YY_FLEX_MINOR_VERSION 5
+#define YY_FLEX_SUBMINOR_VERSION 35
+#if YY_FLEX_SUBMINOR_VERSION > 0
+#define FLEX_BETA
+#endif
+
+/* First, we deal with  platform-specific or compiler-specific issues. */
+
+/* begin standard C headers. */
+#include <stdio.h>
+#include <string.h>
+#include <errno.h>
+#include <stdlib.h>
+
+/* end standard C headers. */
+
+/* flex integer type definitions */
+
+#ifndef FLEXINT_H
+#define FLEXINT_H
+
+/* C99 systems have <inttypes.h>. Non-C99 systems may or may not. */
+
+#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L
+
+/* C99 says to define __STDC_LIMIT_MACROS before including stdint.h,
+ * if you want the limit (max/min) macros for int types. 
+ */
+#ifndef __STDC_LIMIT_MACROS
+#define __STDC_LIMIT_MACROS 1
+#endif
+
+#include <inttypes.h>
+typedef int8_t flex_int8_t;
+typedef uint8_t flex_uint8_t;
+typedef int16_t flex_int16_t;
+typedef uint16_t flex_uint16_t;
+typedef int32_t flex_int32_t;
+typedef uint32_t flex_uint32_t;
+#else
+typedef signed char flex_int8_t;
+typedef short int flex_int16_t;
+typedef int flex_int32_t;
+typedef unsigned char flex_uint8_t; 
+typedef unsigned short int flex_uint16_t;
+typedef unsigned int flex_uint32_t;
+#endif /* ! C99 */
+
+/* Limits of integral types. */
+#ifndef INT8_MIN
+#define INT8_MIN               (-128)
+#endif
+#ifndef INT16_MIN
+#define INT16_MIN              (-32767-1)
+#endif
+#ifndef INT32_MIN
+#define INT32_MIN              (-2147483647-1)
+#endif
+#ifndef INT8_MAX
+#define INT8_MAX               (127)
+#endif
+#ifndef INT16_MAX
+#define INT16_MAX              (32767)
+#endif
+#ifndef INT32_MAX
+#define INT32_MAX              (2147483647)
+#endif
+#ifndef UINT8_MAX
+#define UINT8_MAX              (255U)
+#endif
+#ifndef UINT16_MAX
+#define UINT16_MAX             (65535U)
+#endif
+#ifndef UINT32_MAX
+#define UINT32_MAX             (4294967295U)
+#endif
+
+#endif /* ! FLEXINT_H */
+
+#ifdef __cplusplus
+
+/* The "const" storage-class-modifier is valid. */
+#define YY_USE_CONST
+
+#else	/* ! __cplusplus */
+
+/* C99 requires __STDC__ to be defined as 1. */
+#if defined (__STDC__)
+
+#define YY_USE_CONST
+
+#endif	/* defined (__STDC__) */
+#endif	/* ! __cplusplus */
+
+#ifdef YY_USE_CONST
+#define yyconst const
+#else
+#define yyconst
+#endif
+
+/* Returned upon end-of-file. */
+#define YY_NULL 0
+
+/* Promotes a possibly negative, possibly signed char to an unsigned
+ * integer for use as an array index.  If the signed char is negative,
+ * we want to instead treat it as an 8-bit unsigned char, hence the
+ * double cast.
+ */
+#define YY_SC_TO_UI(c) ((unsigned int) (unsigned char) c)
+
+/* An opaque pointer. */
+#ifndef YY_TYPEDEF_YY_SCANNER_T
+#define YY_TYPEDEF_YY_SCANNER_T
+typedef void* yyscan_t;
+#endif
+
+/* For convenience, these vars (plus the bison vars far below)
+   are macros in the reentrant scanner. */
+#define yyin yyg->yyin_r
+#define yyout yyg->yyout_r
+#define yyextra yyg->yyextra_r
+#define yyleng yyg->yyleng_r
+#define yytext yyg->yytext_r
+#define yylineno (YY_CURRENT_BUFFER_LVALUE->yy_bs_lineno)
+#define yycolumn (YY_CURRENT_BUFFER_LVALUE->yy_bs_column)
+#define yy_flex_debug yyg->yy_flex_debug_r
+
+/* Enter a start condition.  This macro really ought to take a parameter,
+ * but we do it the disgusting crufty way forced on us by the ()-less
+ * definition of BEGIN.
+ */
+#define BEGIN yyg->yy_start = 1 + 2 *
+
+/* Translate the current start state into a value that can be later handed
+ * to BEGIN to return to the state.  The YYSTATE alias is for lex
+ * compatibility.
+ */
+#define YY_START ((yyg->yy_start - 1) / 2)
+#define YYSTATE YY_START
+
+/* Action number for EOF rule of a given start state. */
+#define YY_STATE_EOF(state) (YY_END_OF_BUFFER + state + 1)
+
+/* Special action meaning "start processing a new file". */
+#define YY_NEW_FILE langscan_sh_lex_restart(yyin ,yyscanner )
+
+#define YY_END_OF_BUFFER_CHAR 0
+
+/* Size of default input buffer. */
+#ifndef YY_BUF_SIZE
+#define YY_BUF_SIZE 16384
+#endif
+
+/* The state buf must be large enough to hold one state per character in the main buffer.
+ */
+#define YY_STATE_BUF_SIZE   ((YY_BUF_SIZE + 2) * sizeof(yy_state_type))
+
+#ifndef YY_TYPEDEF_YY_BUFFER_STATE
+#define YY_TYPEDEF_YY_BUFFER_STATE
+typedef struct yy_buffer_state *YY_BUFFER_STATE;
+#endif
+
+#ifndef YY_TYPEDEF_YY_SIZE_T
+#define YY_TYPEDEF_YY_SIZE_T
+typedef size_t yy_size_t;
+#endif
+
+#define EOB_ACT_CONTINUE_SCAN 0
+#define EOB_ACT_END_OF_FILE 1
+#define EOB_ACT_LAST_MATCH 2
+
+    #define YY_LESS_LINENO(n)
+    
+/* Return all but the first "n" matched characters back to the input stream. */
+#define yyless(n) \
+	do \
+		{ \
+		/* Undo effects of setting up yytext. */ \
+        int yyless_macro_arg = (n); \
+        YY_LESS_LINENO(yyless_macro_arg);\
+		*yy_cp = yyg->yy_hold_char; \
+		YY_RESTORE_YY_MORE_OFFSET \
+		yyg->yy_c_buf_p = yy_cp = yy_bp + yyless_macro_arg - YY_MORE_ADJ; \
+		YY_DO_BEFORE_ACTION; /* set up yytext again */ \
+		} \
+	while ( 0 )
+
+#define unput(c) yyunput( c, yyg->yytext_ptr , yyscanner )
+
+#ifndef YY_STRUCT_YY_BUFFER_STATE
+#define YY_STRUCT_YY_BUFFER_STATE
+struct yy_buffer_state
+	{
+	FILE *yy_input_file;
+
+	char *yy_ch_buf;		/* input buffer */
+	char *yy_buf_pos;		/* current position in input buffer */
+
+	/* Size of input buffer in bytes, not including room for EOB
+	 * characters.
+	 */
+	yy_size_t yy_buf_size;
+
+	/* Number of characters read into yy_ch_buf, not including EOB
+	 * characters.
+	 */
+	yy_size_t yy_n_chars;
+
+	/* Whether we "own" the buffer - i.e., we know we created it,
+	 * and can realloc() it to grow it, and should free() it to
+	 * delete it.
+	 */
+	int yy_is_our_buffer;
+
+	/* Whether this is an "interactive" input source; if so, and
+	 * if we're using stdio for input, then we want to use getc()
+	 * instead of fread(), to make sure we stop fetching input after
+	 * each newline.
+	 */
+	int yy_is_interactive;
+
+	/* Whether we're considered to be at the beginning of a line.
+	 * If so, '^' rules will be active on the next match, otherwise
+	 * not.
+	 */
+	int yy_at_bol;
+
+    int yy_bs_lineno; /**< The line count. */
+    int yy_bs_column; /**< The column count. */
+    
+	/* Whether to try to fill the input buffer when we reach the
+	 * end of it.
+	 */
+	int yy_fill_buffer;
+
+	int yy_buffer_status;
+
+#define YY_BUFFER_NEW 0
+#define YY_BUFFER_NORMAL 1
+	/* When an EOF's been seen but there's still some text to process
+	 * then we mark the buffer as YY_EOF_PENDING, to indicate that we
+	 * shouldn't try reading from the input source any more.  We might
+	 * still have a bunch of tokens to match, though, because of
+	 * possible backing-up.
+	 *
+	 * When we actually see the EOF, we change the status to "new"
+	 * (via langscan_sh_lex_restart()), so that the user can continue scanning by
+	 * just pointing yyin at a new input file.
+	 */
+#define YY_BUFFER_EOF_PENDING 2
+
+	};
+#endif /* !YY_STRUCT_YY_BUFFER_STATE */
+
+/* We provide macros for accessing buffer states in case in the
+ * future we want to put the buffer states in a more general
+ * "scanner state".
+ *
+ * Returns the top of the stack, or NULL.
+ */
+#define YY_CURRENT_BUFFER ( yyg->yy_buffer_stack \
+                          ? yyg->yy_buffer_stack[yyg->yy_buffer_stack_top] \
+                          : NULL)
+
+/* Same as previous macro, but useful when we know that the buffer stack is not
+ * NULL or when we need an lvalue. For internal use only.
+ */
+#define YY_CURRENT_BUFFER_LVALUE yyg->yy_buffer_stack[yyg->yy_buffer_stack_top]
+
+void langscan_sh_lex_restart (FILE *input_file ,yyscan_t yyscanner );
+void langscan_sh_lex__switch_to_buffer (YY_BUFFER_STATE new_buffer ,yyscan_t yyscanner );
+YY_BUFFER_STATE langscan_sh_lex__create_buffer (FILE *file,int size ,yyscan_t yyscanner );
+void langscan_sh_lex__delete_buffer (YY_BUFFER_STATE b ,yyscan_t yyscanner );
+void langscan_sh_lex__flush_buffer (YY_BUFFER_STATE b ,yyscan_t yyscanner );
+void langscan_sh_lex_push_buffer_state (YY_BUFFER_STATE new_buffer ,yyscan_t yyscanner );
+void langscan_sh_lex_pop_buffer_state (yyscan_t yyscanner );
+
+static void langscan_sh_lex_ensure_buffer_stack (yyscan_t yyscanner );
+static void langscan_sh_lex__load_buffer_state (yyscan_t yyscanner );
+static void langscan_sh_lex__init_buffer (YY_BUFFER_STATE b,FILE *file ,yyscan_t yyscanner );
+
+#define YY_FLUSH_BUFFER langscan_sh_lex__flush_buffer(YY_CURRENT_BUFFER ,yyscanner)
+
+YY_BUFFER_STATE langscan_sh_lex__scan_buffer (char *base,yy_size_t size ,yyscan_t yyscanner );
+YY_BUFFER_STATE langscan_sh_lex__scan_string (yyconst char *yy_str ,yyscan_t yyscanner );
+YY_BUFFER_STATE langscan_sh_lex__scan_bytes (yyconst char *bytes,yy_size_t len ,yyscan_t yyscanner );
+
+void *langscan_sh_lex_alloc (yy_size_t ,yyscan_t yyscanner );
+void *langscan_sh_lex_realloc (void *,yy_size_t ,yyscan_t yyscanner );
+void langscan_sh_lex_free (void * ,yyscan_t yyscanner );
+
+#define yy_new_buffer langscan_sh_lex__create_buffer
+
+#define yy_set_interactive(is_interactive) \
+	{ \
+	if ( ! YY_CURRENT_BUFFER ){ \
+        langscan_sh_lex_ensure_buffer_stack (yyscanner); \
+		YY_CURRENT_BUFFER_LVALUE =    \
+            langscan_sh_lex__create_buffer(yyin,YY_BUF_SIZE ,yyscanner); \
+	} \
+	YY_CURRENT_BUFFER_LVALUE->yy_is_interactive = is_interactive; \
+	}
+
+#define yy_set_bol(at_bol) \
+	{ \
+	if ( ! YY_CURRENT_BUFFER ){\
+        langscan_sh_lex_ensure_buffer_stack (yyscanner); \
+		YY_CURRENT_BUFFER_LVALUE =    \
+            langscan_sh_lex__create_buffer(yyin,YY_BUF_SIZE ,yyscanner); \
+	} \
+	YY_CURRENT_BUFFER_LVALUE->yy_at_bol = at_bol; \
+	}
+
+#define YY_AT_BOL() (YY_CURRENT_BUFFER_LVALUE->yy_at_bol)
+
+/* Begin user sect3 */
+
+#define langscan_sh_lex_wrap(n) 1
+#define YY_SKIP_YYWRAP
+
+typedef unsigned char YY_CHAR;
+
+typedef int yy_state_type;
+
+#define yytext_ptr yytext_r
+
+static yy_state_type yy_get_previous_state (yyscan_t yyscanner );
+static yy_state_type yy_try_NUL_trans (yy_state_type current_state  ,yyscan_t yyscanner);
+static int yy_get_next_buffer (yyscan_t yyscanner );
+static void yy_fatal_error (yyconst char msg[] ,yyscan_t yyscanner );
+
+/* Done after the current pattern has been matched and before the
+ * corresponding action - sets up yytext.
+ */
+#define YY_DO_BEFORE_ACTION \
+	yyg->yytext_ptr = yy_bp; \
+	yyleng = (size_t) (yy_cp - yy_bp); \
+	yyg->yy_hold_char = *yy_cp; \
+	*yy_cp = '\0'; \
+	yyg->yy_c_buf_p = yy_cp;
+
+#define YY_NUM_RULES 27
+#define YY_END_OF_BUFFER 28
+/* This struct is not used in this scanner,
+   but its presence is necessary. */
+struct yy_trans_info
+	{
+	flex_int32_t yy_verify;
+	flex_int32_t yy_nxt;
+	};
+static yyconst flex_int16_t yy_accept[164] =
+    {   0,
+        0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
+        0,    0,    0,    0,    0,    0,   28,   26,   19,   20,
+       20,   26,    4,   26,   26,   26,   14,   22,   26,   26,
+       26,   26,    7,   12,   26,   16,    6,    6,    6,    6,
+        6,    5,    6,    6,    6,    6,    6,    6,   26,    6,
+        6,   26,    8,   19,   11,   11,   11,   11,   11,   11,
+       10,   15,   13,    2,    2,    2,    2,    2,    2,    2,
+        2,    2,    2,    2,    2,   26,    3,    3,    3,    3,
+        3,    3,    3,    3,    3,    3,    3,    3,    3,    3,
+       19,   17,   20,   25,   23,   14,   23,   23,   23,    9,
+
+        0,   18,    0,    0,   22,    1,   24,   16,    6,    0,
+        6,    6,    6,    6,    6,    6,    6,    6,    6,    0,
+       19,   11,    2,    2,    2,    2,    2,    2,    2,    2,
+        2,    2,    2,    2,    3,    3,    3,    3,    3,    3,
+        3,    3,    3,    3,    3,    3,    3,   17,   23,    0,
+       21,    1,    6,    6,    6,    6,    2,    2,    2,    3,
+        3,    3,    0
+    } ;
+
+static yyconst flex_int32_t yy_ec[256] =
+    {   0,
+        1,    1,    1,    1,    1,    1,    1,    1,    2,    3,
+        1,    1,    4,    1,    1,    1,    1,    1,    1,    1,
+        1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
+        1,    2,    5,    6,    7,    8,    1,    9,   10,   11,
+       12,   13,    1,    1,   14,   15,    1,   16,   16,   16,
+       16,   16,   16,   16,   16,   16,   16,    1,   17,   18,
+       19,   20,   21,   22,   23,   23,   23,   23,   23,   23,
+       23,   23,   23,   23,   23,   23,   23,   23,   23,   23,
+       23,   23,   23,   23,   23,   23,   23,   23,   23,   23,
+        1,   24,    1,    1,   25,   26,   23,   23,   23,   23,
+
+       23,   23,   23,   23,   23,   23,   23,   23,   23,   23,
+       23,   23,   23,   23,   23,   23,   23,   23,   23,   23,
+       23,   23,   27,   28,   29,    1,    1,    1,    1,    1,
+        1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
+        1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
+        1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
+        1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
+        1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
+        1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
+        1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
+
+        1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
+        1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
+        1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
+        1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
+        1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
+        1,    1,    1,    1,    1
+    } ;
+
+static yyconst flex_int32_t yy_meta[30] =
+    {   0,
+        1,    2,    3,    1,    1,    4,    1,    4,    1,    1,
+        2,    1,    1,    2,    2,    2,    1,    1,    1,    1,
+        1,    1,    2,    1,    2,    4,    1,    1,    1
+    } ;
+
+static yyconst flex_int16_t yy_base[175] =
+    {   0,
+        0,  386,   29,    0,   41,   53,   78,    0,   50,   54,
+      363,   58,  107,    0,  135,  161,  391,  394,   62,  394,
+      387,  370,  394,  186,  379,  377,  394,   71,  369,   79,
+       65,    0,  394,  394,  357,    0,  360,  136,  358,   78,
+       51,  394,   54,  354,  212,  122,  131,  133,   73,  350,
+       65,   86,  394,  368,  347,  394,  328,  145,  180,    0,
+      394,  394,  394,    0,  332,  233,  334,    0,  257,  325,
+      197,  135,    0,    0,  312,  309,    0,  332,  326,  304,
+      278,  306,    0,  302,  291,  201,  139,    0,    0,  268,
+      210,    0,  394,  394,  394,  394,  394,  206,  209,  394,
+
+      285,  394,  190,  231,    0,  234,  394,    0,  269,  258,
+      237,  252,  233,  263,  251,    0,  180,  227,  218,  232,
+      228,    0,    0,    0,    0,    0,    0,  244,  254,  279,
+      276,    0,  200,    0,    0,  193,    0,    0,    0,    0,
+      282,  286,  295,  308,    0,  141,    0,    0,  296,  312,
+      394,  394,  221,  320,  127,   82,  310,  326,    0,  314,
+      329,    0,  394,  344,   88,  348,  352,  356,  359,  363,
+      367,  371,  375,  379
+    } ;
+
+static yyconst flex_int16_t yy_def[175] =
+    {   0,
+      163,    1,  163,    3,    1,    1,    1,    7,    1,    1,
+        1,    1,  163,   13,    1,  163,  163,  163,  163,  163,
+      163,  163,  163,  163,  163,  164,  163,  165,  163,  163,
+      163,  166,  163,  163,  163,  167,  168,  168,  168,  168,
+      168,  163,  168,  168,  168,  168,  168,  168,  169,  168,
+      168,  163,  163,  163,  163,  163,   24,  165,   58,   59,
+      163,  163,  163,  170,  170,  170,  170,  170,  170,  170,
+      170,  170,  171,  170,  170,   24,  172,  172,  172,  172,
+      172,  172,  172,  172,  172,  172,  172,  173,  172,  172,
+      163,  174,  163,  163,  163,  163,  163,  163,  163,  163,
+
+      164,  163,  163,  163,   58,  163,  163,  167,  168,  168,
+      168,  168,  168,  168,  168,   45,  168,  168,  168,  163,
+      163,   59,  170,  170,  170,  170,  170,  170,  170,  170,
+      170,   69,  170,  170,  172,  172,  172,  172,  172,  172,
+      172,  172,  172,  172,   84,  172,  172,  174,  163,  163,
+      163,  163,  168,  168,  168,  168,  170,  170,  170,  172,
+      172,  172,    0,  163,  163,  163,  163,  163,  163,  163,
+      163,  163,  163,  163
+    } ;
+
+static yyconst flex_int16_t yy_nxt[424] =
+    {   0,
+       18,   19,   20,   21,   22,   23,   18,   24,   25,   26,
+       27,   18,   18,   18,   18,   28,   29,   30,   18,   31,
+       18,   18,   28,   32,   28,   33,   34,   35,   18,   37,
+       38,   39,   40,   41,   42,   37,   24,   43,   37,   44,
+       37,   37,   37,   37,   45,   46,   47,   37,   48,   37,
+       37,   45,   49,   45,   33,   50,   51,   37,   52,   36,
+       36,   62,  113,   91,   36,   62,   53,   52,   92,  113,
+       52,   52,  103,   94,  110,  109,  119,  110,   53,   54,
+      112,  104,   55,   94,   56,   57,   63,   94,  110,  105,
+       56,   56,  113,   58,   94,   52,  106,   94,   56,   56,
+
+       59,  110,   60,  120,   94,  110,   61,   64,   54,   20,
+       21,   65,   64,   64,   66,   67,   64,   68,   64,   64,
+       64,   64,   69,   70,   71,   64,   72,   64,   64,   69,
+       73,   69,   64,   74,   75,   64,   54,  111,  113,  113,
+       18,  113,   76,  124,   18,  110,  103,  137,  117,  113,
+      110,  113,   52,  124,  110,  104,  110,  137,  137,  110,
+       18,   77,   78,   20,   79,   80,   77,   77,   81,   82,
+       77,   83,   77,   77,   77,   77,   84,   85,   86,   77,
+       87,   77,   77,   84,   88,   84,   77,   89,   90,   77,
+       95,  103,   95,   95,  136,  122,   96,  113,   95,   95,
+
+      104,   97,  122,  110,  122,  124,   95,   95,   98,  137,
+       99,   91,  100,  114,  133,  124,   92,  124,  146,  137,
+      156,  149,  115,  156,  149,  116,  116,  116,  149,  121,
+      149,  149,  150,  149,  116,  110,  116,  125,  111,  125,
+      125,  110,  151,  126,  110,  125,  125,  152,  127,   94,
+      110,   94,  154,  125,  125,  128,  110,  129,  130,  157,
+      110,  153,  155,  109,  114,  109,  157,  131,  157,  157,
+      132,  132,  132,  115,  110,  110,  157,  158,  157,  132,
+      130,  132,  138,  109,  138,  138,  110,  159,  139,  131,
+      138,  138,  110,  140,  102,  137,  143,  160,  138,  138,
+
+      141,  160,  142,  143,  160,  144,  160,  137,  160,  161,
+      160,  149,  144,  150,  137,  145,  145,  145,  149,  162,
+      149,  154,  137,  151,  145,  157,  145,  158,   93,  160,
+      161,  155,  157,  136,  157,  163,  160,  159,  160,  124,
+      162,  124,  124,  110,  101,  101,  101,  101,  107,  107,
+      124,  107,  108,  108,  163,  108,  109,  109,  109,  118,
+      118,  118,  118,  123,  123,   94,  123,  134,  134,  121,
+      134,  135,  135,  110,  135,  147,  147,  110,  147,  148,
+      148,  110,  148,  110,   94,   94,  102,   94,   94,   93,
+      163,   63,   36,   17,  163,  163,  163,  163,  163,  163,
+
+      163,  163,  163,  163,  163,  163,  163,  163,  163,  163,
+      163,  163,  163,  163,  163,  163,  163,  163,  163,  163,
+      163,  163,  163
+    } ;
+
+static yyconst flex_int16_t yy_chk[424] =
+    {   0,
+        1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
+        1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
+        1,    1,    1,    1,    1,    1,    1,    1,    1,    3,
+        3,    3,    3,    3,    3,    3,    3,    3,    3,    3,
+        3,    3,    3,    3,    3,    3,    3,    3,    3,    3,
+        3,    3,    3,    3,    3,    3,    3,    3,    5,    6,
+       10,    9,   43,   19,   12,   10,    5,    9,   19,   41,
+        6,   10,   28,   31,   41,   49,   49,   43,    6,    7,
+       40,   28,    7,   31,    7,    7,   12,   30,   51,  165,
+        7,    7,   51,    7,   52,    7,   30,   30,    7,    7,
+
+        7,   40,    7,   52,   52,  156,    7,   13,   13,   13,
+       13,   13,   13,   13,   13,   13,   13,   13,   13,   13,
+       13,   13,   13,   13,   13,   13,   13,   13,   13,   13,
+       13,   13,   13,   13,   13,   13,   15,   38,   46,   47,
+       15,   48,   15,   72,   15,   46,   58,   87,   47,   47,
+      155,   48,   15,   72,   47,   58,   48,   87,  146,   38,
+       15,   16,   16,   16,   16,   16,   16,   16,   16,   16,
+       16,   16,   16,   16,   16,   16,   16,   16,   16,   16,
+       16,   16,   16,   16,   16,   16,   16,   16,   16,   16,
+       24,  103,   24,   24,  136,   59,   24,  117,   24,   24,
+
+      103,   24,   59,  117,   59,   71,   24,   24,   24,   86,
+       24,   91,   24,   45,   71,   71,   91,  133,   86,   86,
+      119,   98,   45,  153,   99,   45,   45,   45,   98,  121,
+       98,   99,  104,   99,   45,   45,   45,   66,  111,   66,
+       66,  119,  104,   66,  153,   66,   66,  106,   66,  120,
+      118,  106,  115,   66,   66,   66,  113,   66,   69,  128,
+      111,  110,  115,  110,  114,  110,  128,   69,  128,  129,
+       69,   69,   69,  114,  115,  112,  129,  131,  129,   69,
+      130,   69,   81,  110,   81,   81,  114,  131,   81,  130,
+       81,   81,  109,   81,  101,   90,  143,  141,   81,   81,
+
+       81,  142,   81,   84,  141,  143,  141,   85,  142,  144,
+      142,  149,   84,  150,   82,   84,   84,   84,  149,  144,
+      149,  154,   80,  150,   84,  157,   84,  158,   79,  160,
+      161,  154,  157,   78,  157,   76,  160,  158,  160,   75,
+      161,   70,   67,  154,  164,  164,  164,  164,  166,  166,
+       65,  166,  167,  167,   57,  167,  168,  168,  168,  169,
+      169,  169,  169,  170,  170,   55,  170,  171,  171,   54,
+      171,  172,  172,   50,  172,  173,  173,   44,  173,  174,
+      174,   39,  174,   37,   35,   29,   26,   25,   22,   21,
+       17,   11,    2,  163,  163,  163,  163,  163,  163,  163,
+
+      163,  163,  163,  163,  163,  163,  163,  163,  163,  163,
+      163,  163,  163,  163,  163,  163,  163,  163,  163,  163,
+      163,  163,  163
+    } ;
+
+/* The intent behind this definition is that it'll catch
+ * any uses of REJECT which flex missed.
+ */
+#define REJECT reject_used_but_not_detected
+#define yymore() yymore_used_but_not_detected
+#define YY_MORE_ADJ 0
+#define YY_RESTORE_YY_MORE_OFFSET
+/*
+ * shtok.l - a lex rule for shell scripts
+ * 
+ * Copyright (C) 2005 Kenichi Ishibashi <bashi at dream.ie.ariake-nct.ac.jp>
+ *     All rights reserved.
+ *     This is free software with ABSOLUTELY NO WARRANTY.
+ * 
+ * You can redistribute it and/or modify it under the terms of 
+ * the GNU General Public License version 2.
+ */
+
+#include "sh.h"
+
+#define YY_EXTRA_TYPE langscan_sh_lex_extra_t *
+
+#if YY_NULL != 0
+#error "YY_NULL is not 0."
+#endif
+
+#define YY_DECL langscan_sh_token_t langscan_sh_lex_lex(yyscan_t yyscanner)
+
+#define YY_INPUT(buf,result,max_size) \
+  if (!yyextra->eof) { \
+    result = yyextra->user_read(&(yyextra->user_data), (buf), (max_size)); \
+    if (result == 0) \
+      yyextra->eof = 1; \
+  }
+
+#define UPD update_pos(yyextra, yytext, yyleng)
+static void update_pos(langscan_sh_lex_extra_t *, char *, int);
+
+#define report(token) \
+  do { \
+    yyextra->text = yytext; \
+    yyextra->leng = yyleng; \
+    return langscan_sh_##token; \
+  } while (0)
+
+#define PUSH_STATE(state) yy_push_state(state, yyscanner)
+#define POP_STATE yy_pop_state(yyscanner)
+
+static int ident_length(unsigned char *ptr, int max);
+
+static char *heredoc_delimiter;
+static enum { HEREDOC_TAB_NO_STRIP, HEREDOC_TAB_STRIP } heredoc_type;
+static int set_heredoc_delimiter(unsigned char *ptr, int max);
+
+#define INITIAL 0
+#define DQUOTE 1
+#define BQUOTE 2
+#define BRACE_SUBST 3
+#define PAREN_SUBST 4
+#define IN_BRACE 5
+#define HEREDOC_DELIMITER 6
+#define HEREDOC 7
+
+#ifndef YY_NO_UNISTD_H
+/* Special case for "unistd.h", since it is non-ANSI. We include it way
+ * down here because we want the user's section 1 to have been scanned first.
+ * The user has a chance to override it with an option.
+ */
+#include <unistd.h>
+#endif
+
+#ifndef YY_EXTRA_TYPE
+#define YY_EXTRA_TYPE void *
+#endif
+
+/* Holds the entire state of the reentrant scanner. */
+struct yyguts_t
+    {
+
+    /* User-defined. Not touched by flex. */
+    YY_EXTRA_TYPE yyextra_r;
+
+    /* The rest are the same as the globals declared in the non-reentrant scanner. */
+    FILE *yyin_r, *yyout_r;
+    size_t yy_buffer_stack_top; /**< index of top of stack. */
+    size_t yy_buffer_stack_max; /**< capacity of stack. */
+    YY_BUFFER_STATE * yy_buffer_stack; /**< Stack as an array. */
+    char yy_hold_char;
+    yy_size_t yy_n_chars;
+    yy_size_t yyleng_r;
+    char *yy_c_buf_p;
+    int yy_init;
+    int yy_start;
+    int yy_did_buffer_switch_on_eof;
+    int yy_start_stack_ptr;
+    int yy_start_stack_depth;
+    int *yy_start_stack;
+    yy_state_type yy_last_accepting_state;
+    char* yy_last_accepting_cpos;
+
+    int yylineno_r;
+    int yy_flex_debug_r;
+
+    char *yytext_r;
+    int yy_more_flag;
+    int yy_more_len;
+
+    }; /* end struct yyguts_t */
+
+static int yy_init_globals (yyscan_t yyscanner );
+
+int langscan_sh_lex_lex_init (yyscan_t* scanner);
+
+int langscan_sh_lex_lex_init_extra (YY_EXTRA_TYPE user_defined,yyscan_t* scanner);
+
+/* Accessor methods to globals.
+   These are made visible to non-reentrant scanners for convenience. */
+
+int langscan_sh_lex_lex_destroy (yyscan_t yyscanner );
+
+int langscan_sh_lex_get_debug (yyscan_t yyscanner );
+
+void langscan_sh_lex_set_debug (int debug_flag ,yyscan_t yyscanner );
+
+YY_EXTRA_TYPE langscan_sh_lex_get_extra (yyscan_t yyscanner );
+
+void langscan_sh_lex_set_extra (YY_EXTRA_TYPE user_defined ,yyscan_t yyscanner );
+
+FILE *langscan_sh_lex_get_in (yyscan_t yyscanner );
+
+void langscan_sh_lex_set_in  (FILE * in_str ,yyscan_t yyscanner );
+
+FILE *langscan_sh_lex_get_out (yyscan_t yyscanner );
+
+void langscan_sh_lex_set_out  (FILE * out_str ,yyscan_t yyscanner );
+
+yy_size_t langscan_sh_lex_get_leng (yyscan_t yyscanner );
+
+char *langscan_sh_lex_get_text (yyscan_t yyscanner );
+
+int langscan_sh_lex_get_lineno (yyscan_t yyscanner );
+
+void langscan_sh_lex_set_lineno (int line_number ,yyscan_t yyscanner );
+
+/* Macros after this point can all be overridden by user definitions in
+ * section 1.
+ */
+
+#ifndef YY_SKIP_YYWRAP
+#ifdef __cplusplus
+extern "C" int langscan_sh_lex_wrap (yyscan_t yyscanner );
+#else
+extern int langscan_sh_lex_wrap (yyscan_t yyscanner );
+#endif
+#endif
+
+    static void yyunput (int c,char *buf_ptr  ,yyscan_t yyscanner);
+    
+#ifndef yytext_ptr
+static void yy_flex_strncpy (char *,yyconst char *,int ,yyscan_t yyscanner);
+#endif
+
+#ifdef YY_NEED_STRLEN
+static int yy_flex_strlen (yyconst char * ,yyscan_t yyscanner);
+#endif
+
+#ifndef YY_NO_INPUT
+
+#ifdef __cplusplus
+static int yyinput (yyscan_t yyscanner );
+#else
+static int input (yyscan_t yyscanner );
+#endif
+
+#endif
+
+    static void yy_push_state (int new_state ,yyscan_t yyscanner);
+    
+    static void yy_pop_state (yyscan_t yyscanner );
+    
+    static int yy_top_state (yyscan_t yyscanner );
+    
+/* Amount of stuff to slurp up with each read. */
+#ifndef YY_READ_BUF_SIZE
+#define YY_READ_BUF_SIZE 8192
+#endif
+
+/* Copy whatever the last rule matched to the standard output. */
+#ifndef ECHO
+/* This used to be an fputs(), but since the string might contain NUL's,
+ * we now use fwrite().
+ */
+#define ECHO fwrite( yytext, yyleng, 1, yyout )
+#endif
+
+/* Gets input and stuffs it into "buf".  number of characters read, or YY_NULL,
+ * is returned in "result".
+ */
+#ifndef YY_INPUT
+#define YY_INPUT(buf,result,max_size) \
+	if ( YY_CURRENT_BUFFER_LVALUE->yy_is_interactive ) \
+		{ \
+		int c = '*'; \
+		yy_size_t n; \
+		for ( n = 0; n < max_size && \
+			     (c = getc( yyin )) != EOF && c != '\n'; ++n ) \
+			buf[n] = (char) c; \
+		if ( c == '\n' ) \
+			buf[n++] = (char) c; \
+		if ( c == EOF && ferror( yyin ) ) \
+			YY_FATAL_ERROR( "input in flex scanner failed" ); \
+		result = n; \
+		} \
+	else \
+		{ \
+		errno=0; \
+		while ( (result = fread(buf, 1, max_size, yyin))==0 && ferror(yyin)) \
+			{ \
+			if( errno != EINTR) \
+				{ \
+				YY_FATAL_ERROR( "input in flex scanner failed" ); \
+				break; \
+				} \
+			errno=0; \
+			clearerr(yyin); \
+			} \
+		}\
+\
+
+#endif
+
+/* No semi-colon after return; correct usage is to write "yyterminate();" -
+ * we don't want an extra ';' after the "return" because that will cause
+ * some compilers to complain about unreachable statements.
+ */
+#ifndef yyterminate
+#define yyterminate() return YY_NULL
+#endif
+
+/* Number of entries by which start-condition stack grows. */
+#ifndef YY_START_STACK_INCR
+#define YY_START_STACK_INCR 25
+#endif
+
+/* Report a fatal error. */
+#ifndef YY_FATAL_ERROR
+#define YY_FATAL_ERROR(msg) yy_fatal_error( msg , yyscanner)
+#endif
+
+/* end tables serialization structures and prototypes */
+
+/* Default declaration of generated scanner - a define so the user can
+ * easily add parameters.
+ */
+#ifndef YY_DECL
+#define YY_DECL_IS_OURS 1
+
+extern int langscan_sh_lex_lex (yyscan_t yyscanner);
+
+#define YY_DECL int langscan_sh_lex_lex (yyscan_t yyscanner)
+#endif /* !YY_DECL */
+
+/* Code executed at the beginning of each rule, after yytext and yyleng
+ * have been set up.
+ */
+#ifndef YY_USER_ACTION
+#define YY_USER_ACTION
+#endif
+
+/* Code executed at the end of each rule. */
+#ifndef YY_BREAK
+#define YY_BREAK break;
+#endif
+
+#define YY_RULE_SETUP \
+	if ( yyleng > 0 ) \
+		YY_CURRENT_BUFFER_LVALUE->yy_at_bol = \
+				(yytext[yyleng - 1] == '\n'); \
+	YY_USER_ACTION
+
+/** The main scanner function which does all the work.
+ */
+YY_DECL
+{
+	register yy_state_type yy_current_state;
+	register char *yy_cp, *yy_bp;
+	register int yy_act;
+    struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
+
+	if ( !yyg->yy_init )
+		{
+		yyg->yy_init = 1;
+
+#ifdef YY_USER_INIT
+		YY_USER_INIT;
+#endif
+
+		if ( ! yyg->yy_start )
+			yyg->yy_start = 1;	/* first start state */
+
+		if ( ! yyin )
+			yyin = stdin;
+
+		if ( ! yyout )
+			yyout = stdout;
+
+		if ( ! YY_CURRENT_BUFFER ) {
+			langscan_sh_lex_ensure_buffer_stack (yyscanner);
+			YY_CURRENT_BUFFER_LVALUE =
+				langscan_sh_lex__create_buffer(yyin,YY_BUF_SIZE ,yyscanner);
+		}
+
+		langscan_sh_lex__load_buffer_state(yyscanner );
+		}
+
+	while ( 1 )		/* loops until end-of-file is reached */
+		{
+		yy_cp = yyg->yy_c_buf_p;
+
+		/* Support of yytext. */
+		*yy_cp = yyg->yy_hold_char;
+
+		/* yy_bp points to the position in yy_ch_buf of the start of
+		 * the current run.
+		 */
+		yy_bp = yy_cp;
+
+		yy_current_state = yyg->yy_start;
+		yy_current_state += YY_AT_BOL();
+yy_match:
+		do
+			{
+			register YY_CHAR yy_c = yy_ec[YY_SC_TO_UI(*yy_cp)];
+			if ( yy_accept[yy_current_state] )
+				{
+				yyg->yy_last_accepting_state = yy_current_state;
+				yyg->yy_last_accepting_cpos = yy_cp;
+				}
+			while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state )
+				{
+				yy_current_state = (int) yy_def[yy_current_state];
+				if ( yy_current_state >= 164 )
+					yy_c = yy_meta[(unsigned int) yy_c];
+				}
+			yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c];
+			++yy_cp;
+			}
+		while ( yy_base[yy_current_state] != 394 );
+
+yy_find_action:
+		yy_act = yy_accept[yy_current_state];
+		if ( yy_act == 0 )
+			{ /* have to back up */
+			yy_cp = yyg->yy_last_accepting_cpos;
+			yy_current_state = yyg->yy_last_accepting_state;
+			yy_act = yy_accept[yy_current_state];
+			}
+
+		YY_DO_BEFORE_ACTION;
+
+do_action:	/* This label is used only to access EOF actions. */
+
+		switch ( yy_act )
+	{ /* beginning of action switch */
+			case 0: /* must back up */
+			/* undo the effects of YY_DO_BEFORE_ACTION */
+			*yy_cp = yyg->yy_hold_char;
+			yy_cp = yyg->yy_last_accepting_cpos;
+			yy_current_state = yyg->yy_last_accepting_state;
+			goto yy_find_action;
+
+case 1:
+YY_RULE_SETUP
+{
+  if (yytext[yyleng - 1] == '-')
+    heredoc_type = HEREDOC_TAB_STRIP;
+  else
+    heredoc_type = HEREDOC_TAB_NO_STRIP;
+  UPD;
+  PUSH_STATE(HEREDOC_DELIMITER);
+  report(punct);
+}
+	YY_BREAK
+case 2:
+YY_RULE_SETUP
+{
+  int delimiter_leng;
+  delimiter_leng = set_heredoc_delimiter(yytext, yyleng);
+  if (delimiter_leng == -1) YY_FATAL_ERROR("Can't allocate memory");
+  yyless(delimiter_leng);
+	PUSH_STATE(HEREDOC);
+  UPD;
+  report(heredoc_beg);
+}
+	YY_BREAK
+case 3:
+YY_RULE_SETUP
+{
+  int sleng;
+  sleng = 0;
+  if (heredoc_type == HEREDOC_TAB_STRIP) {
+    while (yytext[sleng] == ' ' || yytext[sleng] =='\t') {
+      sleng++;
+      if (sleng >= yyleng) { UPD; report(space); }
+    }
+  }
+  if (strcmp((yytext + sleng), heredoc_delimiter) == 0) { /* end-of-heredoc */
+    free(heredoc_delimiter);
+		POP_STATE;
+		POP_STATE;
+    UPD; report(heredoc_end);
+  }
+  else {
+    UPD; report(string);
+  }
+}
+	YY_BREAK
+case YY_STATE_EOF(HEREDOC):
+{
+  free(heredoc_delimiter);
+  BEGIN(INITIAL);
+  UPD; report(string);
+}
+	YY_BREAK
+case 4:
+YY_RULE_SETUP
+{
+		PUSH_STATE(DQUOTE);
+		UPD; report(punct);
+}
+	YY_BREAK
+case 5:
+YY_RULE_SETUP
+{
+		POP_STATE;
+		UPD; report(punct);
+}
+	YY_BREAK
+case YY_STATE_EOF(DQUOTE):
+{ BEGIN(INITIAL); }
+	YY_BREAK
+case 6:
+/* rule 6 can match eol */
+YY_RULE_SETUP
+{ UPD; report(string); }
+	YY_BREAK
+case 7:
+YY_RULE_SETUP
+{
+		PUSH_STATE(BQUOTE);
+		UPD; report(punct);
+}
+	YY_BREAK
+case 8:
+YY_RULE_SETUP
+{
+		POP_STATE;
+		UPD; report(punct);
+}
+	YY_BREAK
+case YY_STATE_EOF(BQUOTE):
+{ BEGIN(INITIAL); }
+	YY_BREAK
+case 9:
+YY_RULE_SETUP
+{
+		PUSH_STATE(BRACE_SUBST);
+		UPD; report(punct);
+}
+	YY_BREAK
+case 10:
+YY_RULE_SETUP
+{
+		POP_STATE;
+		UPD; report(punct);
+}
+	YY_BREAK
+case 11:
+YY_RULE_SETUP
+{ UPD; report(ident); }
+	YY_BREAK
+case YY_STATE_EOF(BRACE_SUBST):
+{ BEGIN(INITIAL); }
+	YY_BREAK
+case 12:
+YY_RULE_SETUP
+{
+		PUSH_STATE(IN_BRACE);
+		UPD; report(punct);
+}
+	YY_BREAK
+case 13:
+YY_RULE_SETUP
+{
+		POP_STATE;
+		UPD; report(punct);
+}
+	YY_BREAK
+case YY_STATE_EOF(IN_BRACE):
+{ BEGIN(INITIAL); }
+	YY_BREAK
+case 14:
+YY_RULE_SETUP
+{
+		PUSH_STATE(PAREN_SUBST);
+		UPD; report(punct);
+}
+	YY_BREAK
+case 15:
+YY_RULE_SETUP
+{
+		POP_STATE;
+		UPD; report(punct);
+}
+	YY_BREAK
+case YY_STATE_EOF(PAREN_SUBST):
+{ BEGIN(INITIAL); }
+	YY_BREAK
+case 16:
+YY_RULE_SETUP
+{ UPD; report(comment); }
+	YY_BREAK
+case 17:
+YY_RULE_SETUP
+{ UPD; report(comment); }
+	YY_BREAK
+case 18:
+/* rule 18 can match eol */
+YY_RULE_SETUP
+{ UPD; report(string); }
+	YY_BREAK
+case 19:
+YY_RULE_SETUP
+{ UPD; report(space); }
+	YY_BREAK
+case 20:
+/* rule 20 can match eol */
+YY_RULE_SETUP
+{ UPD; report(space); }
+	YY_BREAK
+case 21:
+YY_RULE_SETUP
+{ yyless(ident_length(yytext, yyleng)); UPD; report(fundef); }
+	YY_BREAK
+case 22:
+YY_RULE_SETUP
+{ UPD; report(ident); }
+	YY_BREAK
+case 23:
+YY_RULE_SETUP
+{ UPD; report(ident); }
+	YY_BREAK
+case 24:
+YY_RULE_SETUP
+{ UPD; report(punct); }
+	YY_BREAK
+case 25:
+YY_RULE_SETUP
+{ UPD; report(punct); }
+	YY_BREAK
+case 26:
+YY_RULE_SETUP
+{ UPD; report(punct); }
+	YY_BREAK
+case 27:
+YY_RULE_SETUP
+YY_FATAL_ERROR( "flex scanner jammed" );
+	YY_BREAK
+case YY_STATE_EOF(INITIAL):
+case YY_STATE_EOF(HEREDOC_DELIMITER):
+	yyterminate();
+
+	case YY_END_OF_BUFFER:
+		{
+		/* Amount of text matched not including the EOB char. */
+		int yy_amount_of_matched_text = (int) (yy_cp - yyg->yytext_ptr) - 1;
+
+		/* Undo the effects of YY_DO_BEFORE_ACTION. */
+		*yy_cp = yyg->yy_hold_char;
+		YY_RESTORE_YY_MORE_OFFSET
+
+		if ( YY_CURRENT_BUFFER_LVALUE->yy_buffer_status == YY_BUFFER_NEW )
+			{
+			/* We're scanning a new file or input source.  It's
+			 * possible that this happened because the user
+			 * just pointed yyin at a new source and called
+			 * langscan_sh_lex_lex().  If so, then we have to assure
+			 * consistency between YY_CURRENT_BUFFER and our
+			 * globals.  Here is the right place to do so, because
+			 * this is the first action (other than possibly a
+			 * back-up) that will match for the new input source.
+			 */
+			yyg->yy_n_chars = YY_CURRENT_BUFFER_LVALUE->yy_n_chars;
+			YY_CURRENT_BUFFER_LVALUE->yy_input_file = yyin;
+			YY_CURRENT_BUFFER_LVALUE->yy_buffer_status = YY_BUFFER_NORMAL;
+			}
+
+		/* Note that here we test for yy_c_buf_p "<=" to the position
+		 * of the first EOB in the buffer, since yy_c_buf_p will
+		 * already have been incremented past the NUL character
+		 * (since all states make transitions on EOB to the
+		 * end-of-buffer state).  Contrast this with the test
+		 * in input().
+		 */
+		if ( yyg->yy_c_buf_p <= &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[yyg->yy_n_chars] )
+			{ /* This was really a NUL. */
+			yy_state_type yy_next_state;
+
+			yyg->yy_c_buf_p = yyg->yytext_ptr + yy_amount_of_matched_text;
+
+			yy_current_state = yy_get_previous_state( yyscanner );
+
+			/* Okay, we're now positioned to make the NUL
+			 * transition.  We couldn't have
+			 * yy_get_previous_state() go ahead and do it
+			 * for us because it doesn't know how to deal
+			 * with the possibility of jamming (and we don't
+			 * want to build jamming into it because then it
+			 * will run more slowly).
+			 */
+
+			yy_next_state = yy_try_NUL_trans( yy_current_state , yyscanner);
+
+			yy_bp = yyg->yytext_ptr + YY_MORE_ADJ;
+
+			if ( yy_next_state )
+				{
+				/* Consume the NUL. */
+				yy_cp = ++yyg->yy_c_buf_p;
+				yy_current_state = yy_next_state;
+				goto yy_match;
+				}
+
+			else
+				{
+				yy_cp = yyg->yy_c_buf_p;
+				goto yy_find_action;
+				}
+			}
+
+		else switch ( yy_get_next_buffer( yyscanner ) )
+			{
+			case EOB_ACT_END_OF_FILE:
+				{
+				yyg->yy_did_buffer_switch_on_eof = 0;
+
+				if ( langscan_sh_lex_wrap(yyscanner ) )
+					{
+					/* Note: because we've taken care in
+					 * yy_get_next_buffer() to have set up
+					 * yytext, we can now set up
+					 * yy_c_buf_p so that if some total
+					 * hoser (like flex itself) wants to
+					 * call the scanner after we return the
+					 * YY_NULL, it'll still work - another
+					 * YY_NULL will get returned.
+					 */
+					yyg->yy_c_buf_p = yyg->yytext_ptr + YY_MORE_ADJ;
+
+					yy_act = YY_STATE_EOF(YY_START);
+					goto do_action;
+					}
+
+				else
+					{
+					if ( ! yyg->yy_did_buffer_switch_on_eof )
+						YY_NEW_FILE;
+					}
+				break;
+				}
+
+			case EOB_ACT_CONTINUE_SCAN:
+				yyg->yy_c_buf_p =
+					yyg->yytext_ptr + yy_amount_of_matched_text;
+
+				yy_current_state = yy_get_previous_state( yyscanner );
+
+				yy_cp = yyg->yy_c_buf_p;
+				yy_bp = yyg->yytext_ptr + YY_MORE_ADJ;
+				goto yy_match;
+
+			case EOB_ACT_LAST_MATCH:
+				yyg->yy_c_buf_p =
+				&YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[yyg->yy_n_chars];
+
+				yy_current_state = yy_get_previous_state( yyscanner );
+
+				yy_cp = yyg->yy_c_buf_p;
+				yy_bp = yyg->yytext_ptr + YY_MORE_ADJ;
+				goto yy_find_action;
+			}
+		break;
+		}
+
+	default:
+		YY_FATAL_ERROR(
+			"fatal flex scanner internal error--no action found" );
+	} /* end of action switch */
+		} /* end of scanning one token */
+} /* end of langscan_sh_lex_lex */
+
+/* yy_get_next_buffer - try to read in a new buffer
+ *
+ * Returns a code representing an action:
+ *	EOB_ACT_LAST_MATCH -
+ *	EOB_ACT_CONTINUE_SCAN - continue scanning from current position
+ *	EOB_ACT_END_OF_FILE - end of file
+ */
+static int yy_get_next_buffer (yyscan_t yyscanner)
+{
+    struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
+	register char *dest = YY_CURRENT_BUFFER_LVALUE->yy_ch_buf;
+	register char *source = yyg->yytext_ptr;
+	register int number_to_move, i;
+	int ret_val;
+
+	if ( yyg->yy_c_buf_p > &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[yyg->yy_n_chars + 1] )
+		YY_FATAL_ERROR(
+		"fatal flex scanner internal error--end of buffer missed" );
+
+	if ( YY_CURRENT_BUFFER_LVALUE->yy_fill_buffer == 0 )
+		{ /* Don't try to fill the buffer, so this is an EOF. */
+		if ( yyg->yy_c_buf_p - yyg->yytext_ptr - YY_MORE_ADJ == 1 )
+			{
+			/* We matched a single character, the EOB, so
+			 * treat this as a final EOF.
+			 */
+			return EOB_ACT_END_OF_FILE;
+			}
+
+		else
+			{
+			/* We matched some text prior to the EOB, first
+			 * process it.
+			 */
+			return EOB_ACT_LAST_MATCH;
+			}
+		}
+
+	/* Try to read more data. */
+
+	/* First move last chars to start of buffer. */
+	number_to_move = (int) (yyg->yy_c_buf_p - yyg->yytext_ptr) - 1;
+
+	for ( i = 0; i < number_to_move; ++i )
+		*(dest++) = *(source++);
+
+	if ( YY_CURRENT_BUFFER_LVALUE->yy_buffer_status == YY_BUFFER_EOF_PENDING )
+		/* don't do the read, it's not guaranteed to return an EOF,
+		 * just force an EOF
+		 */
+		YY_CURRENT_BUFFER_LVALUE->yy_n_chars = yyg->yy_n_chars = 0;
+
+	else
+		{
+			yy_size_t num_to_read =
+			YY_CURRENT_BUFFER_LVALUE->yy_buf_size - number_to_move - 1;
+
+		while ( num_to_read <= 0 )
+			{ /* Not enough room in the buffer - grow it. */
+
+			/* just a shorter name for the current buffer */
+			YY_BUFFER_STATE b = YY_CURRENT_BUFFER;
+
+			int yy_c_buf_p_offset =
+				(int) (yyg->yy_c_buf_p - b->yy_ch_buf);
+
+			if ( b->yy_is_our_buffer )
+				{
+				yy_size_t new_size = b->yy_buf_size * 2;
+
+				if ( new_size <= 0 )
+					b->yy_buf_size += b->yy_buf_size / 8;
+				else
+					b->yy_buf_size *= 2;
+
+				b->yy_ch_buf = (char *)
+					/* Include room in for 2 EOB chars. */
+					langscan_sh_lex_realloc((void *) b->yy_ch_buf,b->yy_buf_size + 2 ,yyscanner );
+				}
+			else
+				/* Can't grow it, we don't own it. */
+				b->yy_ch_buf = 0;
+
+			if ( ! b->yy_ch_buf )
+				YY_FATAL_ERROR(
+				"fatal error - scanner input buffer overflow" );
+
+			yyg->yy_c_buf_p = &b->yy_ch_buf[yy_c_buf_p_offset];
+
+			num_to_read = YY_CURRENT_BUFFER_LVALUE->yy_buf_size -
+						number_to_move - 1;
+
+			}
+
+		if ( num_to_read > YY_READ_BUF_SIZE )
+			num_to_read = YY_READ_BUF_SIZE;
+
+		/* Read in more data. */
+		YY_INPUT( (&YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[number_to_move]),
+			yyg->yy_n_chars, num_to_read );
+
+		YY_CURRENT_BUFFER_LVALUE->yy_n_chars = yyg->yy_n_chars;
+		}
+
+	if ( yyg->yy_n_chars == 0 )
+		{
+		if ( number_to_move == YY_MORE_ADJ )
+			{
+			ret_val = EOB_ACT_END_OF_FILE;
+			langscan_sh_lex_restart(yyin  ,yyscanner);
+			}
+
+		else
+			{
+			ret_val = EOB_ACT_LAST_MATCH;
+			YY_CURRENT_BUFFER_LVALUE->yy_buffer_status =
+				YY_BUFFER_EOF_PENDING;
+			}
+		}
+
+	else
+		ret_val = EOB_ACT_CONTINUE_SCAN;
+
+	if ((yy_size_t) (yyg->yy_n_chars + number_to_move) > YY_CURRENT_BUFFER_LVALUE->yy_buf_size) {
+		/* Extend the array by 50%, plus the number we really need. */
+		yy_size_t new_size = yyg->yy_n_chars + number_to_move + (yyg->yy_n_chars >> 1);
+		YY_CURRENT_BUFFER_LVALUE->yy_ch_buf = (char *) langscan_sh_lex_realloc((void *) YY_CURRENT_BUFFER_LVALUE->yy_ch_buf,new_size ,yyscanner );
+		if ( ! YY_CURRENT_BUFFER_LVALUE->yy_ch_buf )
+			YY_FATAL_ERROR( "out of dynamic memory in yy_get_next_buffer()" );
+	}
+
+	yyg->yy_n_chars += number_to_move;
+	YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[yyg->yy_n_chars] = YY_END_OF_BUFFER_CHAR;
+	YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[yyg->yy_n_chars + 1] = YY_END_OF_BUFFER_CHAR;
+
+	yyg->yytext_ptr = &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[0];
+
+	return ret_val;
+}
+
+/* yy_get_previous_state - get the state just before the EOB char was reached */
+
+    static yy_state_type yy_get_previous_state (yyscan_t yyscanner)
+{
+	register yy_state_type yy_current_state;
+	register char *yy_cp;
+    struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
+
+	yy_current_state = yyg->yy_start;
+	yy_current_state += YY_AT_BOL();
+
+	for ( yy_cp = yyg->yytext_ptr + YY_MORE_ADJ; yy_cp < yyg->yy_c_buf_p; ++yy_cp )
+		{
+		register YY_CHAR yy_c = (*yy_cp ? yy_ec[YY_SC_TO_UI(*yy_cp)] : 1);
+		if ( yy_accept[yy_current_state] )
+			{
+			yyg->yy_last_accepting_state = yy_current_state;
+			yyg->yy_last_accepting_cpos = yy_cp;
+			}
+		while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state )
+			{
+			yy_current_state = (int) yy_def[yy_current_state];
+			if ( yy_current_state >= 164 )
+				yy_c = yy_meta[(unsigned int) yy_c];
+			}
+		yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c];
+		}
+
+	return yy_current_state;
+}
+
+/* yy_try_NUL_trans - try to make a transition on the NUL character
+ *
+ * synopsis
+ *	next_state = yy_try_NUL_trans( current_state );
+ */
+    static yy_state_type yy_try_NUL_trans  (yy_state_type yy_current_state , yyscan_t yyscanner)
+{
+	register int yy_is_jam;
+    struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; /* This var may be unused depending upon options. */
+	register char *yy_cp = yyg->yy_c_buf_p;
+
+	register YY_CHAR yy_c = 1;
+	if ( yy_accept[yy_current_state] )
+		{
+		yyg->yy_last_accepting_state = yy_current_state;
+		yyg->yy_last_accepting_cpos = yy_cp;
+		}
+	while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state )
+		{
+		yy_current_state = (int) yy_def[yy_current_state];
+		if ( yy_current_state >= 164 )
+			yy_c = yy_meta[(unsigned int) yy_c];
+		}
+	yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c];
+	yy_is_jam = (yy_current_state == 163);
+
+	return yy_is_jam ? 0 : yy_current_state;
+}
+
+    static void yyunput (int c, register char * yy_bp , yyscan_t yyscanner)
+{
+	register char *yy_cp;
+    struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
+
+    yy_cp = yyg->yy_c_buf_p;
+
+	/* undo effects of setting up yytext */
+	*yy_cp = yyg->yy_hold_char;
+
+	if ( yy_cp < YY_CURRENT_BUFFER_LVALUE->yy_ch_buf + 2 )
+		{ /* need to shift things up to make room */
+		/* +2 for EOB chars. */
+		register yy_size_t number_to_move = yyg->yy_n_chars + 2;
+		register char *dest = &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[
+					YY_CURRENT_BUFFER_LVALUE->yy_buf_size + 2];
+		register char *source =
+				&YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[number_to_move];
+
+		while ( source > YY_CURRENT_BUFFER_LVALUE->yy_ch_buf )
+			*--dest = *--source;
+
+		yy_cp += (int) (dest - source);
+		yy_bp += (int) (dest - source);
+		YY_CURRENT_BUFFER_LVALUE->yy_n_chars =
+			yyg->yy_n_chars = YY_CURRENT_BUFFER_LVALUE->yy_buf_size;
+
+		if ( yy_cp < YY_CURRENT_BUFFER_LVALUE->yy_ch_buf + 2 )
+			YY_FATAL_ERROR( "flex scanner push-back overflow" );
+		}
+
+	*--yy_cp = (char) c;
+
+	yyg->yytext_ptr = yy_bp;
+	yyg->yy_hold_char = *yy_cp;
+	yyg->yy_c_buf_p = yy_cp;
+}
+
+#ifndef YY_NO_INPUT
+#ifdef __cplusplus
+    static int yyinput (yyscan_t yyscanner)
+#else
+    static int input  (yyscan_t yyscanner)
+#endif
+
+{
+	int c;
+    struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
+
+	*yyg->yy_c_buf_p = yyg->yy_hold_char;
+
+	if ( *yyg->yy_c_buf_p == YY_END_OF_BUFFER_CHAR )
+		{
+		/* yy_c_buf_p now points to the character we want to return.
+		 * If this occurs *before* the EOB characters, then it's a
+		 * valid NUL; if not, then we've hit the end of the buffer.
+		 */
+		if ( yyg->yy_c_buf_p < &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[yyg->yy_n_chars] )
+			/* This was really a NUL. */
+			*yyg->yy_c_buf_p = '\0';
+
+		else
+			{ /* need more input */
+			yy_size_t offset = yyg->yy_c_buf_p - yyg->yytext_ptr;
+			++yyg->yy_c_buf_p;
+
+			switch ( yy_get_next_buffer( yyscanner ) )
+				{
+				case EOB_ACT_LAST_MATCH:
+					/* This happens because yy_g_n_b()
+					 * sees that we've accumulated a
+					 * token and flags that we need to
+					 * try matching the token before
+					 * proceeding.  But for input(),
+					 * there's no matching to consider.
+					 * So convert the EOB_ACT_LAST_MATCH
+					 * to EOB_ACT_END_OF_FILE.
+					 */
+
+					/* Reset buffer status. */
+					langscan_sh_lex_restart(yyin ,yyscanner);
+
+					/*FALLTHROUGH*/
+
+				case EOB_ACT_END_OF_FILE:
+					{
+					if ( langscan_sh_lex_wrap(yyscanner ) )
+						return 0;
+
+					if ( ! yyg->yy_did_buffer_switch_on_eof )
+						YY_NEW_FILE;
+#ifdef __cplusplus
+					return yyinput(yyscanner);
+#else
+					return input(yyscanner);
+#endif
+					}
+
+				case EOB_ACT_CONTINUE_SCAN:
+					yyg->yy_c_buf_p = yyg->yytext_ptr + offset;
+					break;
+				}
+			}
+		}
+
+	c = *(unsigned char *) yyg->yy_c_buf_p;	/* cast for 8-bit char's */
+	*yyg->yy_c_buf_p = '\0';	/* preserve yytext */
+	yyg->yy_hold_char = *++yyg->yy_c_buf_p;
+
+	YY_CURRENT_BUFFER_LVALUE->yy_at_bol = (c == '\n');
+
+	return c;
+}
+#endif	/* ifndef YY_NO_INPUT */
+
+/** Immediately switch to a different input stream.
+ * @param input_file A readable stream.
+ * @param yyscanner The scanner object.
+ * @note This function does not reset the start condition to @c INITIAL .
+ */
+    void langscan_sh_lex_restart  (FILE * input_file , yyscan_t yyscanner)
+{
+    struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
+
+	if ( ! YY_CURRENT_BUFFER ){
+        langscan_sh_lex_ensure_buffer_stack (yyscanner);
+		YY_CURRENT_BUFFER_LVALUE =
+            langscan_sh_lex__create_buffer(yyin,YY_BUF_SIZE ,yyscanner);
+	}
+
+	langscan_sh_lex__init_buffer(YY_CURRENT_BUFFER,input_file ,yyscanner);
+	langscan_sh_lex__load_buffer_state(yyscanner );
+}
+
+/** Switch to a different input buffer.
+ * @param new_buffer The new input buffer.
+ * @param yyscanner The scanner object.
+ */
+    void langscan_sh_lex__switch_to_buffer  (YY_BUFFER_STATE  new_buffer , yyscan_t yyscanner)
+{
+    struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
+
+	/* TODO. We should be able to replace this entire function body
+	 * with
+	 *		langscan_sh_lex_pop_buffer_state();
+	 *		langscan_sh_lex_push_buffer_state(new_buffer);
+     */
+	langscan_sh_lex_ensure_buffer_stack (yyscanner);
+	if ( YY_CURRENT_BUFFER == new_buffer )
+		return;
+
+	if ( YY_CURRENT_BUFFER )
+		{
+		/* Flush out information for old buffer. */
+		*yyg->yy_c_buf_p = yyg->yy_hold_char;
+		YY_CURRENT_BUFFER_LVALUE->yy_buf_pos = yyg->yy_c_buf_p;
+		YY_CURRENT_BUFFER_LVALUE->yy_n_chars = yyg->yy_n_chars;
+		}
+
+	YY_CURRENT_BUFFER_LVALUE = new_buffer;
+	langscan_sh_lex__load_buffer_state(yyscanner );
+
+	/* We don't actually know whether we did this switch during
+	 * EOF (langscan_sh_lex_wrap()) processing, but the only time this flag
+	 * is looked at is after langscan_sh_lex_wrap() is called, so it's safe
+	 * to go ahead and always set it.
+	 */
+	yyg->yy_did_buffer_switch_on_eof = 1;
+}
+
+static void langscan_sh_lex__load_buffer_state  (yyscan_t yyscanner)
+{
+    struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
+	yyg->yy_n_chars = YY_CURRENT_BUFFER_LVALUE->yy_n_chars;
+	yyg->yytext_ptr = yyg->yy_c_buf_p = YY_CURRENT_BUFFER_LVALUE->yy_buf_pos;
+	yyin = YY_CURRENT_BUFFER_LVALUE->yy_input_file;
+	yyg->yy_hold_char = *yyg->yy_c_buf_p;
+}
+
+/** Allocate and initialize an input buffer state.
+ * @param file A readable stream.
+ * @param size The character buffer size in bytes. When in doubt, use @c YY_BUF_SIZE.
+ * @param yyscanner The scanner object.
+ * @return the allocated buffer state.
+ */
+    YY_BUFFER_STATE langscan_sh_lex__create_buffer  (FILE * file, int  size , yyscan_t yyscanner)
+{
+	YY_BUFFER_STATE b;
+    
+	b = (YY_BUFFER_STATE) langscan_sh_lex_alloc(sizeof( struct yy_buffer_state ) ,yyscanner );
+	if ( ! b )
+		YY_FATAL_ERROR( "out of dynamic memory in langscan_sh_lex__create_buffer()" );
+
+	b->yy_buf_size = size;
+
+	/* yy_ch_buf has to be 2 characters longer than the size given because
+	 * we need to put in 2 end-of-buffer characters.
+	 */
+	b->yy_ch_buf = (char *) langscan_sh_lex_alloc(b->yy_buf_size + 2 ,yyscanner );
+	if ( ! b->yy_ch_buf )
+		YY_FATAL_ERROR( "out of dynamic memory in langscan_sh_lex__create_buffer()" );
+
+	b->yy_is_our_buffer = 1;
+
+	langscan_sh_lex__init_buffer(b,file ,yyscanner);
+
+	return b;
+}
+
+/** Destroy the buffer.
+ * @param b a buffer created with langscan_sh_lex__create_buffer()
+ * @param yyscanner The scanner object.
+ */
+    void langscan_sh_lex__delete_buffer (YY_BUFFER_STATE  b , yyscan_t yyscanner)
+{
+    struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
+
+	if ( ! b )
+		return;
+
+	if ( b == YY_CURRENT_BUFFER ) /* Not sure if we should pop here. */
+		YY_CURRENT_BUFFER_LVALUE = (YY_BUFFER_STATE) 0;
+
+	if ( b->yy_is_our_buffer )
+		langscan_sh_lex_free((void *) b->yy_ch_buf ,yyscanner );
+
+	langscan_sh_lex_free((void *) b ,yyscanner );
+}
+
+#ifndef __cplusplus
+extern int isatty (int );
+#endif /* __cplusplus */
+    
+/* Initializes or reinitializes a buffer.
+ * This function is sometimes called more than once on the same buffer,
+ * such as during a langscan_sh_lex_restart() or at EOF.
+ */
+    static void langscan_sh_lex__init_buffer  (YY_BUFFER_STATE  b, FILE * file , yyscan_t yyscanner)
+
+{
+	int oerrno = errno;
+    struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
+
+	langscan_sh_lex__flush_buffer(b ,yyscanner);
+
+	b->yy_input_file = file;
+	b->yy_fill_buffer = 1;
+
+    /* If b is the current buffer, then langscan_sh_lex__init_buffer was _probably_
+     * called from langscan_sh_lex_restart() or through yy_get_next_buffer.
+     * In that case, we don't want to reset the lineno or column.
+     */
+    if (b != YY_CURRENT_BUFFER){
+        b->yy_bs_lineno = 1;
+        b->yy_bs_column = 0;
+    }
+
+        b->yy_is_interactive = file ? (isatty( fileno(file) ) > 0) : 0;
+    
+	errno = oerrno;
+}
+
+/** Discard all buffered characters. On the next scan, YY_INPUT will be called.
+ * @param b the buffer state to be flushed, usually @c YY_CURRENT_BUFFER.
+ * @param yyscanner The scanner object.
+ */
+    void langscan_sh_lex__flush_buffer (YY_BUFFER_STATE  b , yyscan_t yyscanner)
+{
+    struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
+	if ( ! b )
+		return;
+
+	b->yy_n_chars = 0;
+
+	/* We always need two end-of-buffer characters.  The first causes
+	 * a transition to the end-of-buffer state.  The second causes
+	 * a jam in that state.
+	 */
+	b->yy_ch_buf[0] = YY_END_OF_BUFFER_CHAR;
+	b->yy_ch_buf[1] = YY_END_OF_BUFFER_CHAR;
+
+	b->yy_buf_pos = &b->yy_ch_buf[0];
+
+	b->yy_at_bol = 1;
+	b->yy_buffer_status = YY_BUFFER_NEW;
+
+	if ( b == YY_CURRENT_BUFFER )
+		langscan_sh_lex__load_buffer_state(yyscanner );
+}
+
+/** Pushes the new state onto the stack. The new state becomes
+ *  the current state. This function will allocate the stack
+ *  if necessary.
+ *  @param new_buffer The new state.
+ *  @param yyscanner The scanner object.
+ */
+void langscan_sh_lex_push_buffer_state (YY_BUFFER_STATE new_buffer , yyscan_t yyscanner)
+{
+    struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
+	if (new_buffer == NULL)
+		return;
+
+	langscan_sh_lex_ensure_buffer_stack(yyscanner);
+
+	/* This block is copied from langscan_sh_lex__switch_to_buffer. */
+	if ( YY_CURRENT_BUFFER )
+		{
+		/* Flush out information for old buffer. */
+		*yyg->yy_c_buf_p = yyg->yy_hold_char;
+		YY_CURRENT_BUFFER_LVALUE->yy_buf_pos = yyg->yy_c_buf_p;
+		YY_CURRENT_BUFFER_LVALUE->yy_n_chars = yyg->yy_n_chars;
+		}
+
+	/* Only push if top exists. Otherwise, replace top. */
+	if (YY_CURRENT_BUFFER)
+		yyg->yy_buffer_stack_top++;
+	YY_CURRENT_BUFFER_LVALUE = new_buffer;
+
+	/* copied from langscan_sh_lex__switch_to_buffer. */
+	langscan_sh_lex__load_buffer_state(yyscanner );
+	yyg->yy_did_buffer_switch_on_eof = 1;
+}
+
+/** Removes and deletes the top of the stack, if present.
+ *  The next element becomes the new top.
+ *  @param yyscanner The scanner object.
+ */
+void langscan_sh_lex_pop_buffer_state (yyscan_t yyscanner)
+{
+    struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
+	if (!YY_CURRENT_BUFFER)
+		return;
+
+	langscan_sh_lex__delete_buffer(YY_CURRENT_BUFFER ,yyscanner);
+	YY_CURRENT_BUFFER_LVALUE = NULL;
+	if (yyg->yy_buffer_stack_top > 0)
+		--yyg->yy_buffer_stack_top;
+
+	if (YY_CURRENT_BUFFER) {
+		langscan_sh_lex__load_buffer_state(yyscanner );
+		yyg->yy_did_buffer_switch_on_eof = 1;
+	}
+}
+
+/* Allocates the stack if it does not exist.
+ *  Guarantees space for at least one push.
+ */
+static void langscan_sh_lex_ensure_buffer_stack (yyscan_t yyscanner)
+{
+	yy_size_t num_to_alloc;
+    struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
+
+	if (!yyg->yy_buffer_stack) {
+
+		/* First allocation is just for 2 elements, since we don't know if this
+		 * scanner will even need a stack. We use 2 instead of 1 to avoid an
+		 * immediate realloc on the next call.
+         */
+		num_to_alloc = 1;
+		yyg->yy_buffer_stack = (struct yy_buffer_state**)langscan_sh_lex_alloc
+								(num_to_alloc * sizeof(struct yy_buffer_state*)
+								, yyscanner);
+		if ( ! yyg->yy_buffer_stack )
+			YY_FATAL_ERROR( "out of dynamic memory in langscan_sh_lex_ensure_buffer_stack()" );
+								  
+		memset(yyg->yy_buffer_stack, 0, num_to_alloc * sizeof(struct yy_buffer_state*));
+				
+		yyg->yy_buffer_stack_max = num_to_alloc;
+		yyg->yy_buffer_stack_top = 0;
+		return;
+	}
+
+	if (yyg->yy_buffer_stack_top >= (yyg->yy_buffer_stack_max) - 1){
+
+		/* Increase the buffer to prepare for a possible push. */
+		int grow_size = 8 /* arbitrary grow size */;
+
+		num_to_alloc = yyg->yy_buffer_stack_max + grow_size;
+		yyg->yy_buffer_stack = (struct yy_buffer_state**)langscan_sh_lex_realloc
+								(yyg->yy_buffer_stack,
+								num_to_alloc * sizeof(struct yy_buffer_state*)
+								, yyscanner);
+		if ( ! yyg->yy_buffer_stack )
+			YY_FATAL_ERROR( "out of dynamic memory in langscan_sh_lex_ensure_buffer_stack()" );
+
+		/* zero only the new slots.*/
+		memset(yyg->yy_buffer_stack + yyg->yy_buffer_stack_max, 0, grow_size * sizeof(struct yy_buffer_state*));
+		yyg->yy_buffer_stack_max = num_to_alloc;
+	}
+}
+
+/** Setup the input buffer state to scan directly from a user-specified character buffer.
+ * @param base the character buffer
+ * @param size the size in bytes of the character buffer
+ * @param yyscanner The scanner object.
+ * @return the newly allocated buffer state object. 
+ */
+YY_BUFFER_STATE langscan_sh_lex__scan_buffer  (char * base, yy_size_t  size , yyscan_t yyscanner)
+{
+	YY_BUFFER_STATE b;
+    
+	if ( size < 2 ||
+	     base[size-2] != YY_END_OF_BUFFER_CHAR ||
+	     base[size-1] != YY_END_OF_BUFFER_CHAR )
+		/* They forgot to leave room for the EOB's. */
+		return 0;
+
+	b = (YY_BUFFER_STATE) langscan_sh_lex_alloc(sizeof( struct yy_buffer_state ) ,yyscanner );
+	if ( ! b )
+		YY_FATAL_ERROR( "out of dynamic memory in langscan_sh_lex__scan_buffer()" );
+
+	b->yy_buf_size = size - 2;	/* "- 2" to take care of EOB's */
+	b->yy_buf_pos = b->yy_ch_buf = base;
+	b->yy_is_our_buffer = 0;
+	b->yy_input_file = 0;
+	b->yy_n_chars = b->yy_buf_size;
+	b->yy_is_interactive = 0;
+	b->yy_at_bol = 1;
+	b->yy_fill_buffer = 0;
+	b->yy_buffer_status = YY_BUFFER_NEW;
+
+	langscan_sh_lex__switch_to_buffer(b ,yyscanner );
+
+	return b;
+}
+
+/** Setup the input buffer state to scan a string. The next call to langscan_sh_lex_lex() will
+ * scan from a @e copy of @a str.
+ * @param yystr a NUL-terminated string to scan
+ * @param yyscanner The scanner object.
+ * @return the newly allocated buffer state object.
+ * @note If you want to scan bytes that may contain NUL values, then use
+ *       langscan_sh_lex__scan_bytes() instead.
+ */
+YY_BUFFER_STATE langscan_sh_lex__scan_string (yyconst char * yystr , yyscan_t yyscanner)
+{
+    
+	return langscan_sh_lex__scan_bytes(yystr,strlen(yystr) ,yyscanner);
+}
+
+/** Setup the input buffer state to scan the given bytes. The next call to langscan_sh_lex_lex() will
+ * scan from a @e copy of @a bytes.
+ * @param bytes the byte buffer to scan
+ * @param len the number of bytes in the buffer pointed to by @a bytes.
+ * @param yyscanner The scanner object.
+ * @return the newly allocated buffer state object.
+ */
+YY_BUFFER_STATE langscan_sh_lex__scan_bytes  (yyconst char * yybytes, yy_size_t  _yybytes_len , yyscan_t yyscanner)
+{
+	YY_BUFFER_STATE b;
+	char *buf;
+	yy_size_t n, i;
+    
+	/* Get memory for full buffer, including space for trailing EOB's. */
+	n = _yybytes_len + 2;
+	buf = (char *) langscan_sh_lex_alloc(n ,yyscanner );
+	if ( ! buf )
+		YY_FATAL_ERROR( "out of dynamic memory in langscan_sh_lex__scan_bytes()" );
+
+	for ( i = 0; i < _yybytes_len; ++i )
+		buf[i] = yybytes[i];
+
+	buf[_yybytes_len] = buf[_yybytes_len+1] = YY_END_OF_BUFFER_CHAR;
+
+	b = langscan_sh_lex__scan_buffer(buf,n ,yyscanner);
+	if ( ! b )
+		YY_FATAL_ERROR( "bad buffer in langscan_sh_lex__scan_bytes()" );
+
+	/* It's okay to grow etc. this buffer, and we should throw it
+	 * away when we're done.
+	 */
+	b->yy_is_our_buffer = 1;
+
+	return b;
+}
+
+    static void yy_push_state (int  new_state , yyscan_t yyscanner)
+{
+    struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
+	if ( yyg->yy_start_stack_ptr >= yyg->yy_start_stack_depth )
+		{
+		yy_size_t new_size;
+
+		yyg->yy_start_stack_depth += YY_START_STACK_INCR;
+		new_size = yyg->yy_start_stack_depth * sizeof( int );
+
+		if ( ! yyg->yy_start_stack )
+			yyg->yy_start_stack = (int *) langscan_sh_lex_alloc(new_size ,yyscanner );
+
+		else
+			yyg->yy_start_stack = (int *) langscan_sh_lex_realloc((void *) yyg->yy_start_stack,new_size ,yyscanner );
+
+		if ( ! yyg->yy_start_stack )
+			YY_FATAL_ERROR( "out of memory expanding start-condition stack" );
+		}
+
+	yyg->yy_start_stack[yyg->yy_start_stack_ptr++] = YY_START;
+
+	BEGIN(new_state);
+}
+
+    static void yy_pop_state  (yyscan_t yyscanner)
+{
+    struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
+	if ( --yyg->yy_start_stack_ptr < 0 )
+		YY_FATAL_ERROR( "start-condition stack underflow" );
+
+	BEGIN(yyg->yy_start_stack[yyg->yy_start_stack_ptr]);
+}
+
+    static int yy_top_state  (yyscan_t yyscanner)
+{
+    struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
+	return yyg->yy_start_stack[yyg->yy_start_stack_ptr - 1];
+}
+
+#ifndef YY_EXIT_FAILURE
+#define YY_EXIT_FAILURE 2
+#endif
+
+static void yy_fatal_error (yyconst char* msg , yyscan_t yyscanner)
+{
+    	(void) fprintf( stderr, "%s\n", msg );
+	exit( YY_EXIT_FAILURE );
+}
+
+/* Redefine yyless() so it works in section 3 code. */
+
+#undef yyless
+#define yyless(n) \
+	do \
+		{ \
+		/* Undo effects of setting up yytext. */ \
+        int yyless_macro_arg = (n); \
+        YY_LESS_LINENO(yyless_macro_arg);\
+		yytext[yyleng] = yyg->yy_hold_char; \
+		yyg->yy_c_buf_p = yytext + yyless_macro_arg; \
+		yyg->yy_hold_char = *yyg->yy_c_buf_p; \
+		*yyg->yy_c_buf_p = '\0'; \
+		yyleng = yyless_macro_arg; \
+		} \
+	while ( 0 )
+
+/* Accessor  methods (get/set functions) to struct members. */
+
+/** Get the user-defined data for this scanner.
+ * @param yyscanner The scanner object.
+ */
+YY_EXTRA_TYPE langscan_sh_lex_get_extra  (yyscan_t yyscanner)
+{
+    struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
+    return yyextra;
+}
+
+/** Get the current line number.
+ * @param yyscanner The scanner object.
+ */
+int langscan_sh_lex_get_lineno  (yyscan_t yyscanner)
+{
+    struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
+    
+        if (! YY_CURRENT_BUFFER)
+            return 0;
+    
+    return yylineno;
+}
+
+/** Get the current column number.
+ * @param yyscanner The scanner object.
+ */
+int langscan_sh_lex_get_column  (yyscan_t yyscanner)
+{
+    struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
+    
+        if (! YY_CURRENT_BUFFER)
+            return 0;
+    
+    return yycolumn;
+}
+
+/** Get the input stream.
+ * @param yyscanner The scanner object.
+ */
+FILE *langscan_sh_lex_get_in  (yyscan_t yyscanner)
+{
+    struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
+    return yyin;
+}
+
+/** Get the output stream.
+ * @param yyscanner The scanner object.
+ */
+FILE *langscan_sh_lex_get_out  (yyscan_t yyscanner)
+{
+    struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
+    return yyout;
+}
+
+/** Get the length of the current token.
+ * @param yyscanner The scanner object.
+ */
+yy_size_t langscan_sh_lex_get_leng  (yyscan_t yyscanner)
+{
+    struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
+    return yyleng;
+}
+
+/** Get the current token.
+ * @param yyscanner The scanner object.
+ */
+
+char *langscan_sh_lex_get_text  (yyscan_t yyscanner)
+{
+    struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
+    return yytext;
+}
+
+/** Set the user-defined data. This data is never touched by the scanner.
+ * @param user_defined The data to be associated with this scanner.
+ * @param yyscanner The scanner object.
+ */
+void langscan_sh_lex_set_extra (YY_EXTRA_TYPE  user_defined , yyscan_t yyscanner)
+{
+    struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
+    yyextra = user_defined ;
+}
+
+/** Set the current line number.
+ * @param line_number
+ * @param yyscanner The scanner object.
+ */
+void langscan_sh_lex_set_lineno (int  line_number , yyscan_t yyscanner)
+{
+    struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
+
+        /* lineno is only valid if an input buffer exists. */
+        if (! YY_CURRENT_BUFFER )
+           yy_fatal_error( "langscan_sh_lex_set_lineno called with no buffer" , yyscanner); 
+    
+    yylineno = line_number;
+}
+
+/** Set the current column.
+ * @param line_number
+ * @param yyscanner The scanner object.
+ */
+void langscan_sh_lex_set_column (int  column_no , yyscan_t yyscanner)
+{
+    struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
+
+        /* column is only valid if an input buffer exists. */
+        if (! YY_CURRENT_BUFFER )
+           yy_fatal_error( "langscan_sh_lex_set_column called with no buffer" , yyscanner); 
+    
+    yycolumn = column_no;
+}
+
+/** Set the input stream. This does not discard the current
+ * input buffer.
+ * @param in_str A readable stream.
+ * @param yyscanner The scanner object.
+ * @see langscan_sh_lex__switch_to_buffer
+ */
+void langscan_sh_lex_set_in (FILE *  in_str , yyscan_t yyscanner)
+{
+    struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
+    yyin = in_str ;
+}
+
+void langscan_sh_lex_set_out (FILE *  out_str , yyscan_t yyscanner)
+{
+    struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
+    yyout = out_str ;
+}
+
+int langscan_sh_lex_get_debug  (yyscan_t yyscanner)
+{
+    struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
+    return yy_flex_debug;
+}
+
+void langscan_sh_lex_set_debug (int  bdebug , yyscan_t yyscanner)
+{
+    struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
+    yy_flex_debug = bdebug ;
+}
+
+/* Accessor methods for yylval and yylloc */
+
+/* User-visible API */
+
+/* langscan_sh_lex_lex_init is special because it creates the scanner itself, so it is
+ * the ONLY reentrant function that doesn't take the scanner as the last argument.
+ * That's why we explicitly handle the declaration, instead of using our macros.
+ */
+
+int langscan_sh_lex_lex_init(yyscan_t* ptr_yy_globals)
+
+{
+    if (ptr_yy_globals == NULL){
+        errno = EINVAL;
+        return 1;
+    }
+
+    *ptr_yy_globals = (yyscan_t) langscan_sh_lex_alloc ( sizeof( struct yyguts_t ), NULL );
+
+    if (*ptr_yy_globals == NULL){
+        errno = ENOMEM;
+        return 1;
+    }
+
+    /* By setting to 0xAA, we expose bugs in yy_init_globals. Leave at 0x00 for releases. */
+    memset(*ptr_yy_globals,0x00,sizeof(struct yyguts_t));
+
+    return yy_init_globals ( *ptr_yy_globals );
+}
+
+/* langscan_sh_lex_lex_init_extra has the same functionality as langscan_sh_lex_lex_init, but follows the
+ * convention of taking the scanner as the last argument. Note however, that
+ * this is a *pointer* to a scanner, as it will be allocated by this call (and
+ * is the reason, too, why this function also must handle its own declaration).
+ * The user defined value in the first argument will be available to langscan_sh_lex_alloc in
+ * the yyextra field.
+ */
+
+int langscan_sh_lex_lex_init_extra(YY_EXTRA_TYPE yy_user_defined,yyscan_t* ptr_yy_globals )
+
+{
+    struct yyguts_t dummy_yyguts;
+
+    langscan_sh_lex_set_extra (yy_user_defined, &dummy_yyguts);
+
+    if (ptr_yy_globals == NULL){
+        errno = EINVAL;
+        return 1;
+    }
+	
+    *ptr_yy_globals = (yyscan_t) langscan_sh_lex_alloc ( sizeof( struct yyguts_t ), &dummy_yyguts );
+	
+    if (*ptr_yy_globals == NULL){
+        errno = ENOMEM;
+        return 1;
+    }
+    
+    /* By setting to 0xAA, we expose bugs in
+    yy_init_globals. Leave at 0x00 for releases. */
+    memset(*ptr_yy_globals,0x00,sizeof(struct yyguts_t));
+    
+    langscan_sh_lex_set_extra (yy_user_defined, *ptr_yy_globals);
+    
+    return yy_init_globals ( *ptr_yy_globals );
+}
+
+static int yy_init_globals (yyscan_t yyscanner)
+{
+    struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
+    /* Initialization is the same as for the non-reentrant scanner.
+     * This function is called from langscan_sh_lex_lex_destroy(), so don't allocate here.
+     */
+
+    yyg->yy_buffer_stack = 0;
+    yyg->yy_buffer_stack_top = 0;
+    yyg->yy_buffer_stack_max = 0;
+    yyg->yy_c_buf_p = (char *) 0;
+    yyg->yy_init = 0;
+    yyg->yy_start = 0;
+
+    yyg->yy_start_stack_ptr = 0;
+    yyg->yy_start_stack_depth = 0;
+    yyg->yy_start_stack =  NULL;
+
+/* Defined in main.c */
+#ifdef YY_STDINIT
+    yyin = stdin;
+    yyout = stdout;
+#else
+    yyin = (FILE *) 0;
+    yyout = (FILE *) 0;
+#endif
+
+    /* For future reference: Set errno on error, since we are called by
+     * langscan_sh_lex_lex_init()
+     */
+    return 0;
+}
+
+/* langscan_sh_lex_lex_destroy is for both reentrant and non-reentrant scanners. */
+int langscan_sh_lex_lex_destroy  (yyscan_t yyscanner)
+{
+    struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
+
+    /* Pop the buffer stack, destroying each element. */
+	while(YY_CURRENT_BUFFER){
+		langscan_sh_lex__delete_buffer(YY_CURRENT_BUFFER ,yyscanner );
+		YY_CURRENT_BUFFER_LVALUE = NULL;
+		langscan_sh_lex_pop_buffer_state(yyscanner);
+	}
+
+	/* Destroy the stack itself. */
+	langscan_sh_lex_free(yyg->yy_buffer_stack ,yyscanner);
+	yyg->yy_buffer_stack = NULL;
+
+    /* Destroy the start condition stack. */
+        langscan_sh_lex_free(yyg->yy_start_stack ,yyscanner );
+        yyg->yy_start_stack = NULL;
+
+    /* Reset the globals. This is important in a non-reentrant scanner so the next time
+     * langscan_sh_lex_lex() is called, initialization will occur. */
+    yy_init_globals( yyscanner);
+
+    /* Destroy the main struct (reentrant only). */
+    langscan_sh_lex_free ( yyscanner , yyscanner );
+    yyscanner = NULL;
+    return 0;
+}
+
+/*
+ * Internal utility routines.
+ */
+
+#ifndef yytext_ptr
+static void yy_flex_strncpy (char* s1, yyconst char * s2, int n , yyscan_t yyscanner)
+{
+	register int i;
+	for ( i = 0; i < n; ++i )
+		s1[i] = s2[i];
+}
+#endif
+
+#ifdef YY_NEED_STRLEN
+static int yy_flex_strlen (yyconst char * s , yyscan_t yyscanner)
+{
+	register int n;
+	for ( n = 0; s[n]; ++n )
+		;
+
+	return n;
+}
+#endif
+
+void *langscan_sh_lex_alloc (yy_size_t  size , yyscan_t yyscanner)
+{
+	return (void *) malloc( size );
+}
+
+void *langscan_sh_lex_realloc  (void * ptr, yy_size_t  size , yyscan_t yyscanner)
+{
+	/* The cast to (char *) in the following accommodates both
+	 * implementations that use char* generic pointers, and those
+	 * that use void* generic pointers.  It works with the latter
+	 * because both ANSI C and C++ allow castless assignment from
+	 * any pointer type to void*, and deal with argument conversions
+	 * as though doing an assignment.
+	 */
+	return (void *) realloc( (char *) ptr, size );
+}
+
+void langscan_sh_lex_free (void * ptr , yyscan_t yyscanner)
+{
+	free( (char *) ptr );	/* see langscan_sh_lex_realloc() for (char *) cast */
+}
+
+#define YYTABLES_NAME "yytables"
+
+static void update_pos(
+  langscan_sh_lex_extra_t *extra,
+  char *text,
+  int leng)
+{
+  int i, j;
+  extra->beg_byteno = extra->end_byteno;
+  extra->beg_lineno = extra->end_lineno;
+  extra->beg_columnno = extra->end_columnno;
+  j = 0;
+  for (i = 0; i < leng; i++) {
+    if (text[i] == '\n') {
+      extra->end_lineno++;
+      j = i + 1;
+      extra->end_columnno = 0;
+    }
+  }
+  extra->end_columnno += leng - j;
+  extra->end_byteno += leng;
+}
+
+static int ident_length(unsigned char *ptr, int max)
+{
+  int len = 0;
+  while (0 < max &&
+         (('0' <= *ptr && *ptr <= '9') ||
+          ('A' <= *ptr && *ptr <= 'Z') ||
+          ('a' <= *ptr && *ptr <= 'z') ||
+          *ptr == '_' || *ptr == '-' || *ptr == '.')) {
+    ptr++;
+    len++;
+    max--;
+  }
+  return len;
+}
+
+static int set_heredoc_delimiter(unsigned char *ptr, int max)
+{
+  char *dst, quote_char;
+  int in_quote, len;
+  heredoc_delimiter = malloc(max + 1);
+  if (heredoc_delimiter == NULL) return -1;
+  dst = heredoc_delimiter;
+  len = 0;
+  in_quote = 0;
+  while (len < max) {
+    if (in_quote == 0) {  /* unquoted delimiter */
+      if (*ptr == '\'' || *ptr == '\"') {
+        quote_char = *ptr;
+        in_quote = 1;
+        ptr++;
+        if (++len >= max) break;
+        continue;
+      }
+      if (*ptr == ' ' || *ptr == '\t') break;
+      if (*ptr == '\\') {
+        ptr++;
+        if (++len >= max) break;
+      }
+    }
+    else {                /* quoted delimiter */
+      if (*ptr == quote_char) {
+        in_quote = 0;
+        ptr++;
+        if (len++ >= max) break;
+        continue;
+      }
+    }
+    *dst++ = *ptr++;
+    len++;
+  }
+  *dst = '\0';
+  return len;
+}
+
+langscan_sh_tokenizer_t *langscan_sh_make_tokenizer(
+  size_t (*user_read)(void **user_data_p, char *buf, size_t maxlen),
+  void *user_data)
+{
+  langscan_sh_tokenizer_t *tokenizer;
+  langscan_sh_lex_extra_t *extra;
+  tokenizer = (langscan_sh_tokenizer_t *)malloc(sizeof(langscan_sh_tokenizer_t));
+  if (tokenizer == NULL)
+    return NULL;
+  extra = (langscan_sh_lex_extra_t *)malloc(sizeof(langscan_sh_lex_extra_t));
+  if (extra == NULL)
+    return NULL;
+  extra->user_read = user_read;
+  extra->user_data = user_data;
+  extra->beg_lineno = 1;
+  extra->beg_columnno = 0;
+  extra->beg_byteno = 0;
+  extra->end_lineno = 1;
+  extra->end_columnno = 0;
+  extra->end_byteno = 0;
+  extra->eof = 0;
+  tokenizer->extra = extra;
+  langscan_sh_lex_lex_init(&tokenizer->scanner);
+  langscan_sh_lex_set_extra(extra, tokenizer->scanner);
+  return tokenizer;
+}
+
+langscan_sh_token_t langscan_sh_get_token(langscan_sh_tokenizer_t *tokenizer) 
+{
+  return langscan_sh_lex_lex(tokenizer->scanner);
+}
+
+void langscan_sh_free_tokenizer(langscan_sh_tokenizer_t *tokenizer) 
+{
+  langscan_sh_lex_extra_t *extra = langscan_sh_lex_get_extra(tokenizer->scanner);
+  free((void *)extra);
+  langscan_sh_lex_lex_destroy(tokenizer->scanner);
+  free((void *)tokenizer);
+}
+
+user_read_t langscan_sh_tokenizer_get_user_read(langscan_sh_tokenizer_t *tokenizer)
+{
+  return tokenizer->extra->user_read;
+}
+
+void *langscan_sh_tokenizer_get_user_data(langscan_sh_tokenizer_t *tokenizer)
+{
+  return tokenizer->extra->user_data;
+}
+
+const char *langscan_sh_token_name(langscan_sh_token_t token)
+{
+  static char *token_names[] = {
+    "*eof*",
+#define LANGSCAN_SH_TOKEN(name) #name,
+    LANGSCAN_SH_TOKEN_LIST
+#undef LANGSCAN_SH_TOKEN
+  };
+
+  return token_names[token];
+}
+