tcc 0.1.0

data/ext/tcc/tcc-0.9.26/tccpp.c

@@ -0,0 +1,3128 @@
+/*
+ *  TCC - Tiny C Compiler
+ * 
+ *  Copyright (c) 2001-2004 Fabrice Bellard
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+#include "tcc.h"
+
+/********************************************************/
+/* global variables */
+
+ST_DATA int tok_flags;
+/* additional informations about token */
+#define TOK_FLAG_BOL   0x0001 /* beginning of line before */
+#define TOK_FLAG_BOF   0x0002 /* beginning of file before */
+#define TOK_FLAG_ENDIF 0x0004 /* a endif was found matching starting #ifdef */
+#define TOK_FLAG_EOF   0x0008 /* end of file */
+
+ST_DATA int parse_flags;
+#define PARSE_FLAG_PREPROCESS 0x0001 /* activate preprocessing */
+#define PARSE_FLAG_TOK_NUM    0x0002 /* return numbers instead of TOK_PPNUM */
+#define PARSE_FLAG_LINEFEED   0x0004 /* line feed is returned as a
+                                        token. line feed is also
+                                        returned at eof */
+#define PARSE_FLAG_ASM_COMMENTS 0x0008 /* '#' can be used for line comment */
+#define PARSE_FLAG_SPACES     0x0010 /* next() returns space tokens (for -E) */
+
+ST_DATA struct BufferedFile *file;
+ST_DATA int ch, tok;
+ST_DATA CValue tokc;
+ST_DATA const int *macro_ptr;
+ST_DATA CString tokcstr; /* current parsed string, if any */
+
+/* display benchmark infos */
+ST_DATA int total_lines;
+ST_DATA int total_bytes;
+ST_DATA int tok_ident;
+ST_DATA TokenSym **table_ident;
+
+/* ------------------------------------------------------------------------- */
+
+static int *macro_ptr_allocated;
+static const int *unget_saved_macro_ptr;
+static int unget_saved_buffer[TOK_MAX_SIZE + 1];
+static int unget_buffer_enabled;
+static TokenSym *hash_ident[TOK_HASH_SIZE];
+static char token_buf[STRING_MAX_SIZE + 1];
+/* true if isid(c) || isnum(c) */
+static unsigned char isidnum_table[256-CH_EOF];
+
+static const char tcc_keywords[] = 
+#define DEF(id, str) str "\0"
+#include "tcctok.h"
+#undef DEF
+;
+
+/* WARNING: the content of this string encodes token numbers */
+static const unsigned char tok_two_chars[] =
+    "<=\236>=\235!=\225&&\240||\241++\244--\242==\224<<\1>>\2+=\253"
+    "-=\255*=\252/=\257%=\245&=\246^=\336|=\374->\313..\250##\266";
+
+struct macro_level {
+    struct macro_level *prev;
+    const int *p;
+};
+
+ST_FUNC void next_nomacro(void);
+static void next_nomacro_spc(void);
+static void macro_subst(
+    TokenString *tok_str,
+    Sym **nested_list,
+    const int *macro_str,
+    struct macro_level **can_read_stream
+    );
+
+ST_FUNC void skip(int c)
+{
+    if (tok != c)
+        tcc_error("'%c' expected (got \"%s\")", c, get_tok_str(tok, &tokc));
+    next();
+}
+
+ST_FUNC void expect(const char *msg)
+{
+    tcc_error("%s expected", msg);
+}
+
+/* ------------------------------------------------------------------------- */
+/* CString handling */
+static void cstr_realloc(CString *cstr, int new_size)
+{
+    int size;
+    void *data;
+
+    size = cstr->size_allocated;
+    if (size == 0)
+        size = 8; /* no need to allocate a too small first string */
+    while (size < new_size)
+        size = size * 2;
+    data = tcc_realloc(cstr->data_allocated, size);
+    cstr->data_allocated = data;
+    cstr->size_allocated = size;
+    cstr->data = data;
+}
+
+/* add a byte */
+ST_FUNC void cstr_ccat(CString *cstr, int ch)
+{
+    int size;
+    size = cstr->size + 1;
+    if (size > cstr->size_allocated)
+        cstr_realloc(cstr, size);
+    ((unsigned char *)cstr->data)[size - 1] = ch;
+    cstr->size = size;
+}
+
+ST_FUNC void cstr_cat(CString *cstr, const char *str)
+{
+    int c;
+    for(;;) {
+        c = *str;
+        if (c == '\0')
+            break;
+        cstr_ccat(cstr, c);
+        str++;
+    }
+}
+
+/* add a wide char */
+ST_FUNC void cstr_wccat(CString *cstr, int ch)
+{
+    int size;
+    size = cstr->size + sizeof(nwchar_t);
+    if (size > cstr->size_allocated)
+        cstr_realloc(cstr, size);
+    *(nwchar_t *)(((unsigned char *)cstr->data) + size - sizeof(nwchar_t)) = ch;
+    cstr->size = size;
+}
+
+ST_FUNC void cstr_new(CString *cstr)
+{
+    memset(cstr, 0, sizeof(CString));
+}
+
+/* free string and reset it to NULL */
+ST_FUNC void cstr_free(CString *cstr)
+{
+    tcc_free(cstr->data_allocated);
+    cstr_new(cstr);
+}
+
+/* reset string to empty */
+ST_FUNC void cstr_reset(CString *cstr)
+{
+    cstr->size = 0;
+}
+
+/* XXX: unicode ? */
+static void add_char(CString *cstr, int c)
+{
+    if (c == '\'' || c == '\"' || c == '\\') {
+        /* XXX: could be more precise if char or string */
+        cstr_ccat(cstr, '\\');
+    }
+    if (c >= 32 && c <= 126) {
+        cstr_ccat(cstr, c);
+    } else {
+        cstr_ccat(cstr, '\\');
+        if (c == '\n') {
+            cstr_ccat(cstr, 'n');
+        } else {
+            cstr_ccat(cstr, '0' + ((c >> 6) & 7));
+            cstr_ccat(cstr, '0' + ((c >> 3) & 7));
+            cstr_ccat(cstr, '0' + (c & 7));
+        }
+    }
+}
+
+/* ------------------------------------------------------------------------- */
+/* allocate a new token */
+static TokenSym *tok_alloc_new(TokenSym **pts, const char *str, int len)
+{
+    TokenSym *ts, **ptable;
+    int i;
+
+    if (tok_ident >= SYM_FIRST_ANOM) 
+        tcc_error("memory full");
+
+    /* expand token table if needed */
+    i = tok_ident - TOK_IDENT;
+    if ((i % TOK_ALLOC_INCR) == 0) {
+        ptable = tcc_realloc(table_ident, (i + TOK_ALLOC_INCR) * sizeof(TokenSym *));
+        table_ident = ptable;
+    }
+
+    ts = tcc_malloc(sizeof(TokenSym) + len);
+    table_ident[i] = ts;
+    ts->tok = tok_ident++;
+    ts->sym_define = NULL;
+    ts->sym_label = NULL;
+    ts->sym_struct = NULL;
+    ts->sym_identifier = NULL;
+    ts->len = len;
+    ts->hash_next = NULL;
+    memcpy(ts->str, str, len);
+    ts->str[len] = '\0';
+    *pts = ts;
+    return ts;
+}
+
+#define TOK_HASH_INIT 1
+#define TOK_HASH_FUNC(h, c) ((h) * 263 + (c))
+
+/* find a token and add it if not found */
+ST_FUNC TokenSym *tok_alloc(const char *str, int len)
+{
+    TokenSym *ts, **pts;
+    int i;
+    unsigned int h;
+    
+    h = TOK_HASH_INIT;
+    for(i=0;i<len;i++)
+        h = TOK_HASH_FUNC(h, ((unsigned char *)str)[i]);
+    h &= (TOK_HASH_SIZE - 1);
+
+    pts = &hash_ident[h];
+    for(;;) {
+        ts = *pts;
+        if (!ts)
+            break;
+        if (ts->len == len && !memcmp(ts->str, str, len))
+            return ts;
+        pts = &(ts->hash_next);
+    }
+    return tok_alloc_new(pts, str, len);
+}
+
+/* XXX: buffer overflow */
+/* XXX: float tokens */
+ST_FUNC char *get_tok_str(int v, CValue *cv)
+{
+    static char buf[STRING_MAX_SIZE + 1];
+    static CString cstr_buf;
+    CString *cstr;
+    char *p;
+    int i, len;
+
+    /* NOTE: to go faster, we give a fixed buffer for small strings */
+    cstr_reset(&cstr_buf);
+    cstr_buf.data = buf;
+    cstr_buf.size_allocated = sizeof(buf);
+    p = buf;
+
+    switch(v) {
+    case TOK_CINT:
+    case TOK_CUINT:
+        /* XXX: not quite exact, but only useful for testing */
+        sprintf(p, "%u", cv->ui);
+        break;
+    case TOK_CLLONG:
+    case TOK_CULLONG:
+        /* XXX: not quite exact, but only useful for testing  */
+#ifdef _WIN32
+        sprintf(p, "%u", (unsigned)cv->ull);
+#else
+        sprintf(p, "%Lu", cv->ull);
+#endif
+        break;
+    case TOK_LCHAR:
+        cstr_ccat(&cstr_buf, 'L');
+    case TOK_CCHAR:
+        cstr_ccat(&cstr_buf, '\'');
+        add_char(&cstr_buf, cv->i);
+        cstr_ccat(&cstr_buf, '\'');
+        cstr_ccat(&cstr_buf, '\0');
+        break;
+    case TOK_PPNUM:
+        cstr = cv->cstr;
+        len = cstr->size - 1;
+        for(i=0;i<len;i++)
+            add_char(&cstr_buf, ((unsigned char *)cstr->data)[i]);
+        cstr_ccat(&cstr_buf, '\0');
+        break;
+    case TOK_LSTR:
+        cstr_ccat(&cstr_buf, 'L');
+    case TOK_STR:
+        cstr = cv->cstr;
+        cstr_ccat(&cstr_buf, '\"');
+        if (v == TOK_STR) {
+            len = cstr->size - 1;
+            for(i=0;i<len;i++)
+                add_char(&cstr_buf, ((unsigned char *)cstr->data)[i]);
+        } else {
+            len = (cstr->size / sizeof(nwchar_t)) - 1;
+            for(i=0;i<len;i++)
+                add_char(&cstr_buf, ((nwchar_t *)cstr->data)[i]);
+        }
+        cstr_ccat(&cstr_buf, '\"');
+        cstr_ccat(&cstr_buf, '\0');
+        break;
+    case TOK_LT:
+        v = '<';
+        goto addv;
+    case TOK_GT:
+        v = '>';
+        goto addv;
+    case TOK_DOTS:
+        return strcpy(p, "...");
+    case TOK_A_SHL:
+        return strcpy(p, "<<=");
+    case TOK_A_SAR:
+        return strcpy(p, ">>=");
+    default:
+        if (v < TOK_IDENT) {
+            /* search in two bytes table */
+            const unsigned char *q = tok_two_chars;
+            while (*q) {
+                if (q[2] == v) {
+                    *p++ = q[0];
+                    *p++ = q[1];
+                    *p = '\0';
+                    return buf;
+                }
+                q += 3;
+            }
+        addv:
+            *p++ = v;
+            *p = '\0';
+        } else if (v < tok_ident) {
+            return table_ident[v - TOK_IDENT]->str;
+        } else if (v >= SYM_FIRST_ANOM) {
+            /* special name for anonymous symbol */
+            sprintf(p, "L.%u", v - SYM_FIRST_ANOM);
+        } else {
+            /* should never happen */
+            return NULL;
+        }
+        break;
+    }
+    return cstr_buf.data;
+}
+
+/* fill input buffer and peek next char */
+static int tcc_peekc_slow(BufferedFile *bf)
+{
+    int len;
+    /* only tries to read if really end of buffer */
+    if (bf->buf_ptr >= bf->buf_end) {
+        if (bf->fd != -1) {
+#if defined(PARSE_DEBUG)
+            len = 8;
+#else
+            len = IO_BUF_SIZE;
+#endif
+            len = read(bf->fd, bf->buffer, len);
+            if (len < 0)
+                len = 0;
+        } else {
+            len = 0;
+        }
+        total_bytes += len;
+        bf->buf_ptr = bf->buffer;
+        bf->buf_end = bf->buffer + len;
+        *bf->buf_end = CH_EOB;
+    }
+    if (bf->buf_ptr < bf->buf_end) {
+        return bf->buf_ptr[0];
+    } else {
+        bf->buf_ptr = bf->buf_end;
+        return CH_EOF;
+    }
+}
+
+/* return the current character, handling end of block if necessary
+   (but not stray) */
+ST_FUNC int handle_eob(void)
+{
+    return tcc_peekc_slow(file);
+}
+
+/* read next char from current input file and handle end of input buffer */
+ST_INLN void inp(void)
+{
+    ch = *(++(file->buf_ptr));
+    /* end of buffer/file handling */
+    if (ch == CH_EOB)
+        ch = handle_eob();
+}
+
+/* handle '\[\r]\n' */
+static int handle_stray_noerror(void)
+{
+    while (ch == '\\') {
+        inp();
+        if (ch == '\n') {
+            file->line_num++;
+            inp();
+        } else if (ch == '\r') {
+            inp();
+            if (ch != '\n')
+                goto fail;
+            file->line_num++;
+            inp();
+        } else {
+        fail:
+            return 1;
+        }
+    }
+    return 0;
+}
+
+static void handle_stray(void)
+{
+    if (handle_stray_noerror())
+        tcc_error("stray '\\' in program");
+}
+
+/* skip the stray and handle the \\n case. Output an error if
+   incorrect char after the stray */
+static int handle_stray1(uint8_t *p)
+{
+    int c;
+
+    if (p >= file->buf_end) {
+        file->buf_ptr = p;
+        c = handle_eob();
+        p = file->buf_ptr;
+        if (c == '\\')
+            goto parse_stray;
+    } else {
+    parse_stray:
+        file->buf_ptr = p;
+        ch = *p;
+        handle_stray();
+        p = file->buf_ptr;
+        c = *p;
+    }
+    return c;
+}
+
+/* handle just the EOB case, but not stray */
+#define PEEKC_EOB(c, p)\
+{\
+    p++;\
+    c = *p;\
+    if (c == '\\') {\
+        file->buf_ptr = p;\
+        c = handle_eob();\
+        p = file->buf_ptr;\
+    }\
+}
+
+/* handle the complicated stray case */
+#define PEEKC(c, p)\
+{\
+    p++;\
+    c = *p;\
+    if (c == '\\') {\
+        c = handle_stray1(p);\
+        p = file->buf_ptr;\
+    }\
+}
+
+/* input with '\[\r]\n' handling. Note that this function cannot
+   handle other characters after '\', so you cannot call it inside
+   strings or comments */
+ST_FUNC void minp(void)
+{
+    inp();
+    if (ch == '\\') 
+        handle_stray();
+}
+
+
+/* single line C++ comments */
+static uint8_t *parse_line_comment(uint8_t *p)
+{
+    int c;
+
+    p++;
+    for(;;) {
+        c = *p;
+    redo:
+        if (c == '\n' || c == CH_EOF) {
+            break;
+        } else if (c == '\\') {
+            file->buf_ptr = p;
+            c = handle_eob();
+            p = file->buf_ptr;
+            if (c == '\\') {
+                PEEKC_EOB(c, p);
+                if (c == '\n') {
+                    file->line_num++;
+                    PEEKC_EOB(c, p);
+                } else if (c == '\r') {
+                    PEEKC_EOB(c, p);
+                    if (c == '\n') {
+                        file->line_num++;
+                        PEEKC_EOB(c, p);
+                    }
+                }
+            } else {
+                goto redo;
+            }
+        } else {
+            p++;
+        }
+    }
+    return p;
+}
+
+/* C comments */
+ST_FUNC uint8_t *parse_comment(uint8_t *p)
+{
+    int c;
+    
+    p++;
+    for(;;) {
+        /* fast skip loop */
+        for(;;) {
+            c = *p;
+            if (c == '\n' || c == '*' || c == '\\')
+                break;
+            p++;
+            c = *p;
+            if (c == '\n' || c == '*' || c == '\\')
+                break;
+            p++;
+        }
+        /* now we can handle all the cases */
+        if (c == '\n') {
+            file->line_num++;
+            p++;
+        } else if (c == '*') {
+            p++;
+            for(;;) {
+                c = *p;
+                if (c == '*') {
+                    p++;
+                } else if (c == '/') {
+                    goto end_of_comment;
+                } else if (c == '\\') {
+                    file->buf_ptr = p;
+                    c = handle_eob();
+                    p = file->buf_ptr;
+                    if (c == '\\') {
+                        /* skip '\[\r]\n', otherwise just skip the stray */
+                        while (c == '\\') {
+                            PEEKC_EOB(c, p);
+                            if (c == '\n') {
+                                file->line_num++;
+                                PEEKC_EOB(c, p);
+                            } else if (c == '\r') {
+                                PEEKC_EOB(c, p);
+                                if (c == '\n') {
+                                    file->line_num++;
+                                    PEEKC_EOB(c, p);
+                                }
+                            } else {
+                                goto after_star;
+                            }
+                        }
+                    }
+                } else {
+                    break;
+                }
+            }
+        after_star: ;
+        } else {
+            /* stray, eob or eof */
+            file->buf_ptr = p;
+            c = handle_eob();
+            p = file->buf_ptr;
+            if (c == CH_EOF) {
+                tcc_error("unexpected end of file in comment");
+            } else if (c == '\\') {
+                p++;
+            }
+        }
+    }
+ end_of_comment:
+    p++;
+    return p;
+}
+
+#define cinp minp
+
+static inline void skip_spaces(void)
+{
+    while (is_space(ch))
+        cinp();
+}
+
+static inline int check_space(int t, int *spc) 
+{
+    if (is_space(t)) {
+        if (*spc) 
+            return 1;
+        *spc = 1;
+    } else 
+        *spc = 0;
+    return 0;
+}
+
+/* parse a string without interpreting escapes */
+static uint8_t *parse_pp_string(uint8_t *p,
+                                int sep, CString *str)
+{
+    int c;
+    p++;
+    for(;;) {
+        c = *p;
+        if (c == sep) {
+            break;
+        } else if (c == '\\') {
+            file->buf_ptr = p;
+            c = handle_eob();
+            p = file->buf_ptr;
+            if (c == CH_EOF) {
+            unterminated_string:
+                /* XXX: indicate line number of start of string */
+                tcc_error("missing terminating %c character", sep);
+            } else if (c == '\\') {
+                /* escape : just skip \[\r]\n */
+                PEEKC_EOB(c, p);
+                if (c == '\n') {
+                    file->line_num++;
+                    p++;
+                } else if (c == '\r') {
+                    PEEKC_EOB(c, p);
+                    if (c != '\n')
+                        expect("'\n' after '\r'");
+                    file->line_num++;
+                    p++;
+                } else if (c == CH_EOF) {
+                    goto unterminated_string;
+                } else {
+                    if (str) {
+                        cstr_ccat(str, '\\');
+                        cstr_ccat(str, c);
+                    }
+                    p++;
+                }
+            }
+        } else if (c == '\n') {
+            file->line_num++;
+            goto add_char;
+        } else if (c == '\r') {
+            PEEKC_EOB(c, p);
+            if (c != '\n') {
+                if (str)
+                    cstr_ccat(str, '\r');
+            } else {
+                file->line_num++;
+                goto add_char;
+            }
+        } else {
+        add_char:
+            if (str)
+                cstr_ccat(str, c);
+            p++;
+        }
+    }
+    p++;
+    return p;
+}
+
+/* skip block of text until #else, #elif or #endif. skip also pairs of
+   #if/#endif */
+static void preprocess_skip(void)
+{
+    int a, start_of_line, c, in_warn_or_error;
+    uint8_t *p;
+
+    p = file->buf_ptr;
+    a = 0;
+redo_start:
+    start_of_line = 1;
+    in_warn_or_error = 0;
+    for(;;) {
+    redo_no_start:
+        c = *p;
+        switch(c) {
+        case ' ':
+        case '\t':
+        case '\f':
+        case '\v':
+        case '\r':
+            p++;
+            goto redo_no_start;
+        case '\n':
+            file->line_num++;
+            p++;
+            goto redo_start;
+        case '\\':
+            file->buf_ptr = p;
+            c = handle_eob();
+            if (c == CH_EOF) {
+                expect("#endif");
+            } else if (c == '\\') {
+                ch = file->buf_ptr[0];
+                handle_stray_noerror();
+            }
+            p = file->buf_ptr;
+            goto redo_no_start;
+        /* skip strings */
+        case '\"':
+        case '\'':
+            if (in_warn_or_error)
+                goto _default;
+            p = parse_pp_string(p, c, NULL);
+            break;
+        /* skip comments */
+        case '/':
+            if (in_warn_or_error)
+                goto _default;
+            file->buf_ptr = p;
+            ch = *p;
+            minp();
+            p = file->buf_ptr;
+            if (ch == '*') {
+                p = parse_comment(p);
+            } else if (ch == '/') {
+                p = parse_line_comment(p);
+            }
+            break;
+        case '#':
+            p++;
+            if (start_of_line) {
+                file->buf_ptr = p;
+                next_nomacro();
+                p = file->buf_ptr;
+                if (a == 0 && 
+                    (tok == TOK_ELSE || tok == TOK_ELIF || tok == TOK_ENDIF))
+                    goto the_end;
+                if (tok == TOK_IF || tok == TOK_IFDEF || tok == TOK_IFNDEF)
+                    a++;
+                else if (tok == TOK_ENDIF)
+                    a--;
+                else if( tok == TOK_ERROR || tok == TOK_WARNING)
+                    in_warn_or_error = 1;
+                else if (tok == TOK_LINEFEED)
+                    goto redo_start;
+            }
+            break;
+_default:
+        default:
+            p++;
+            break;
+        }
+        start_of_line = 0;
+    }
+ the_end: ;
+    file->buf_ptr = p;
+}
+
+/* ParseState handling */
+
+/* XXX: currently, no include file info is stored. Thus, we cannot display
+   accurate messages if the function or data definition spans multiple
+   files */
+
+/* save current parse state in 's' */
+ST_FUNC void save_parse_state(ParseState *s)
+{
+    s->line_num = file->line_num;
+    s->macro_ptr = macro_ptr;
+    s->tok = tok;
+    s->tokc = tokc;
+}
+
+/* restore parse state from 's' */
+ST_FUNC void restore_parse_state(ParseState *s)
+{
+    file->line_num = s->line_num;
+    macro_ptr = s->macro_ptr;
+    tok = s->tok;
+    tokc = s->tokc;
+}
+
+/* return the number of additional 'ints' necessary to store the
+   token */
+static inline int tok_ext_size(int t)
+{
+    switch(t) {
+        /* 4 bytes */
+    case TOK_CINT:
+    case TOK_CUINT:
+    case TOK_CCHAR:
+    case TOK_LCHAR:
+    case TOK_CFLOAT:
+    case TOK_LINENUM:
+        return 1;
+    case TOK_STR:
+    case TOK_LSTR:
+    case TOK_PPNUM:
+        tcc_error("unsupported token");
+        return 1;
+    case TOK_CDOUBLE:
+    case TOK_CLLONG:
+    case TOK_CULLONG:
+        return 2;
+    case TOK_CLDOUBLE:
+        return LDOUBLE_SIZE / 4;
+    default:
+        return 0;
+    }
+}
+
+/* token string handling */
+
+ST_INLN void tok_str_new(TokenString *s)
+{
+    s->str = NULL;
+    s->len = 0;
+    s->allocated_len = 0;
+    s->last_line_num = -1;
+}
+
+ST_FUNC void tok_str_free(int *str)
+{
+    tcc_free(str);
+}
+
+static int *tok_str_realloc(TokenString *s)
+{
+    int *str, len;
+
+    if (s->allocated_len == 0) {
+        len = 8;
+    } else {
+        len = s->allocated_len * 2;
+    }
+    str = tcc_realloc(s->str, len * sizeof(int));
+    s->allocated_len = len;
+    s->str = str;
+    return str;
+}
+
+ST_FUNC void tok_str_add(TokenString *s, int t)
+{
+    int len, *str;
+
+    len = s->len;
+    str = s->str;
+    if (len >= s->allocated_len)
+        str = tok_str_realloc(s);
+    str[len++] = t;
+    s->len = len;
+}
+
+static void tok_str_add2(TokenString *s, int t, CValue *cv)
+{
+    int len, *str;
+
+    len = s->len;
+    str = s->str;
+
+    /* allocate space for worst case */
+    if (len + TOK_MAX_SIZE > s->allocated_len)
+        str = tok_str_realloc(s);
+    str[len++] = t;
+    switch(t) {
+    case TOK_CINT:
+    case TOK_CUINT:
+    case TOK_CCHAR:
+    case TOK_LCHAR:
+    case TOK_CFLOAT:
+    case TOK_LINENUM:
+        str[len++] = cv->tab[0];
+        break;
+    case TOK_PPNUM:
+    case TOK_STR:
+    case TOK_LSTR:
+        {
+            int nb_words;
+            CString *cstr;
+
+            nb_words = (sizeof(CString) + cv->cstr->size + 3) >> 2;
+            while ((len + nb_words) > s->allocated_len)
+                str = tok_str_realloc(s);
+            cstr = (CString *)(str + len);
+            cstr->data = NULL;
+            cstr->size = cv->cstr->size;
+            cstr->data_allocated = NULL;
+            cstr->size_allocated = cstr->size;
+            memcpy((char *)cstr + sizeof(CString), 
+                   cv->cstr->data, cstr->size);
+            len += nb_words;
+        }
+        break;
+    case TOK_CDOUBLE:
+    case TOK_CLLONG:
+    case TOK_CULLONG:
+#if LDOUBLE_SIZE == 8
+    case TOK_CLDOUBLE:
+#endif
+        str[len++] = cv->tab[0];
+        str[len++] = cv->tab[1];
+        break;
+#if LDOUBLE_SIZE == 12
+    case TOK_CLDOUBLE:
+        str[len++] = cv->tab[0];
+        str[len++] = cv->tab[1];
+        str[len++] = cv->tab[2];
+#elif LDOUBLE_SIZE == 16
+    case TOK_CLDOUBLE:
+        str[len++] = cv->tab[0];
+        str[len++] = cv->tab[1];
+        str[len++] = cv->tab[2];
+        str[len++] = cv->tab[3];
+#elif LDOUBLE_SIZE != 8
+#error add long double size support
+#endif
+        break;
+    default:
+        break;
+    }
+    s->len = len;
+}
+
+/* add the current parse token in token string 's' */
+ST_FUNC void tok_str_add_tok(TokenString *s)
+{
+    CValue cval;
+
+    /* save line number info */
+    if (file->line_num != s->last_line_num) {
+        s->last_line_num = file->line_num;
+        cval.i = s->last_line_num;
+        tok_str_add2(s, TOK_LINENUM, &cval);
+    }
+    tok_str_add2(s, tok, &tokc);
+}
+
+/* get a token from an integer array and increment pointer
+   accordingly. we code it as a macro to avoid pointer aliasing. */
+static inline void TOK_GET(int *t, const int **pp, CValue *cv)
+{
+    const int *p = *pp;
+    int n, *tab;
+
+    tab = cv->tab;
+    switch(*t = *p++) {
+    case TOK_CINT:
+    case TOK_CUINT:
+    case TOK_CCHAR:
+    case TOK_LCHAR:
+    case TOK_CFLOAT:
+    case TOK_LINENUM:
+        tab[0] = *p++;
+        break;
+    case TOK_STR:
+    case TOK_LSTR:
+    case TOK_PPNUM:
+        cv->cstr = (CString *)p;
+        cv->cstr->data = (char *)p + sizeof(CString);
+        p += (sizeof(CString) + cv->cstr->size + 3) >> 2;
+        break;
+    case TOK_CDOUBLE:
+    case TOK_CLLONG:
+    case TOK_CULLONG:
+        n = 2;
+        goto copy;
+    case TOK_CLDOUBLE:
+#if LDOUBLE_SIZE == 16
+        n = 4;
+#elif LDOUBLE_SIZE == 12
+        n = 3;
+#elif LDOUBLE_SIZE == 8
+        n = 2;
+#else
+# error add long double size support
+#endif
+    copy:
+        do
+            *tab++ = *p++;
+        while (--n);
+        break;
+    default:
+        break;
+    }
+    *pp = p;
+}
+
+static int macro_is_equal(const int *a, const int *b)
+{
+    char buf[STRING_MAX_SIZE + 1];
+    CValue cv;
+    int t;
+    while (*a && *b) {
+        TOK_GET(&t, &a, &cv);
+        pstrcpy(buf, sizeof buf, get_tok_str(t, &cv));
+        TOK_GET(&t, &b, &cv);
+        if (strcmp(buf, get_tok_str(t, &cv)))
+            return 0;
+    }
+    return !(*a || *b);
+}
+
+/* defines handling */
+ST_INLN void define_push(int v, int macro_type, int *str, Sym *first_arg)
+{
+    Sym *s;
+
+    s = define_find(v);
+    if (s && !macro_is_equal(s->d, str))
+        tcc_warning("%s redefined", get_tok_str(v, NULL));
+
+    s = sym_push2(&define_stack, v, macro_type, 0);
+    s->d = str;
+    s->next = first_arg;
+    table_ident[v - TOK_IDENT]->sym_define = s;
+}
+
+/* undefined a define symbol. Its name is just set to zero */
+ST_FUNC void define_undef(Sym *s)
+{
+    int v;
+    v = s->v;
+    if (v >= TOK_IDENT && v < tok_ident)
+        table_ident[v - TOK_IDENT]->sym_define = NULL;
+    s->v = 0;
+}
+
+ST_INLN Sym *define_find(int v)
+{
+    v -= TOK_IDENT;
+    if ((unsigned)v >= (unsigned)(tok_ident - TOK_IDENT))
+        return NULL;
+    return table_ident[v]->sym_define;
+}
+
+/* free define stack until top reaches 'b' */
+ST_FUNC void free_defines(Sym *b)
+{
+    Sym *top, *top1;
+    int v;
+
+    top = define_stack;
+    while (top != b) {
+        top1 = top->prev;
+        /* do not free args or predefined defines */
+        if (top->d)
+            tok_str_free(top->d);
+        v = top->v;
+        if (v >= TOK_IDENT && v < tok_ident)
+            table_ident[v - TOK_IDENT]->sym_define = NULL;
+        sym_free(top);
+        top = top1;
+    }
+    define_stack = b;
+}
+
+/* label lookup */
+ST_FUNC Sym *label_find(int v)
+{
+    v -= TOK_IDENT;
+    if ((unsigned)v >= (unsigned)(tok_ident - TOK_IDENT))
+        return NULL;
+    return table_ident[v]->sym_label;
+}
+
+ST_FUNC Sym *label_push(Sym **ptop, int v, int flags)
+{
+    Sym *s, **ps;
+    s = sym_push2(ptop, v, 0, 0);
+    s->r = flags;
+    ps = &table_ident[v - TOK_IDENT]->sym_label;
+    if (ptop == &global_label_stack) {
+        /* modify the top most local identifier, so that
+           sym_identifier will point to 's' when popped */
+        while (*ps != NULL)
+            ps = &(*ps)->prev_tok;
+    }
+    s->prev_tok = *ps;
+    *ps = s;
+    return s;
+}
+
+/* pop labels until element last is reached. Look if any labels are
+   undefined. Define symbols if '&&label' was used. */
+ST_FUNC void label_pop(Sym **ptop, Sym *slast)
+{
+    Sym *s, *s1;
+    for(s = *ptop; s != slast; s = s1) {
+        s1 = s->prev;
+        if (s->r == LABEL_DECLARED) {
+            tcc_warning("label '%s' declared but not used", get_tok_str(s->v, NULL));
+        } else if (s->r == LABEL_FORWARD) {
+                tcc_error("label '%s' used but not defined",
+                      get_tok_str(s->v, NULL));
+        } else {
+            if (s->c) {
+                /* define corresponding symbol. A size of
+                   1 is put. */
+                put_extern_sym(s, cur_text_section, s->jnext, 1);
+            }
+        }
+        /* remove label */
+        table_ident[s->v - TOK_IDENT]->sym_label = s->prev_tok;
+        sym_free(s);
+    }
+    *ptop = slast;
+}
+
+/* eval an expression for #if/#elif */
+static int expr_preprocess(void)
+{
+    int c, t;
+    TokenString str;
+    
+    tok_str_new(&str);
+    while (tok != TOK_LINEFEED && tok != TOK_EOF) {
+        next(); /* do macro subst */
+        if (tok == TOK_DEFINED) {
+            next_nomacro();
+            t = tok;
+            if (t == '(') 
+                next_nomacro();
+            c = define_find(tok) != 0;
+            if (t == '(')
+                next_nomacro();
+            tok = TOK_CINT;
+            tokc.i = c;
+        } else if (tok >= TOK_IDENT) {
+            /* if undefined macro */
+            tok = TOK_CINT;
+            tokc.i = 0;
+        }
+        tok_str_add_tok(&str);
+    }
+    tok_str_add(&str, -1); /* simulate end of file */
+    tok_str_add(&str, 0);
+    /* now evaluate C constant expression */
+    macro_ptr = str.str;
+    next();
+    c = expr_const();
+    macro_ptr = NULL;
+    tok_str_free(str.str);
+    return c != 0;
+}
+
+#if defined(PARSE_DEBUG) || defined(PP_DEBUG)
+static void tok_print(int *str)
+{
+    int t;
+    CValue cval;
+
+    printf("<");
+    while (1) {
+        TOK_GET(&t, &str, &cval);
+        if (!t)
+            break;
+        printf("%s", get_tok_str(t, &cval));
+    }
+    printf(">\n");
+}
+#endif
+
+/* parse after #define */
+ST_FUNC void parse_define(void)
+{
+    Sym *s, *first, **ps;
+    int v, t, varg, is_vaargs, spc;
+    TokenString str;
+    
+    v = tok;
+    if (v < TOK_IDENT)
+        tcc_error("invalid macro name '%s'", get_tok_str(tok, &tokc));
+    /* XXX: should check if same macro (ANSI) */
+    first = NULL;
+    t = MACRO_OBJ;
+    /* '(' must be just after macro definition for MACRO_FUNC */
+    next_nomacro_spc();
+    if (tok == '(') {
+        next_nomacro();
+        ps = &first;
+        while (tok != ')') {
+            varg = tok;
+            next_nomacro();
+            is_vaargs = 0;
+            if (varg == TOK_DOTS) {
+                varg = TOK___VA_ARGS__;
+                is_vaargs = 1;
+            } else if (tok == TOK_DOTS && gnu_ext) {
+                is_vaargs = 1;
+                next_nomacro();
+            }
+            if (varg < TOK_IDENT)
+                tcc_error("badly punctuated parameter list");
+            s = sym_push2(&define_stack, varg | SYM_FIELD, is_vaargs, 0);
+            *ps = s;
+            ps = &s->next;
+            if (tok != ',')
+                break;
+            next_nomacro();
+        }
+        if (tok == ')')
+            next_nomacro_spc();
+        t = MACRO_FUNC;
+    }
+    tok_str_new(&str);
+    spc = 2;
+    /* EOF testing necessary for '-D' handling */
+    while (tok != TOK_LINEFEED && tok != TOK_EOF) {
+        /* remove spaces around ## and after '#' */        
+        if (TOK_TWOSHARPS == tok) {
+            if (1 == spc)
+                --str.len;
+            spc = 2;
+        } else if ('#' == tok) {
+            spc = 2;
+        } else if (check_space(tok, &spc)) {
+            goto skip;
+        }
+        tok_str_add2(&str, tok, &tokc);
+    skip:
+        next_nomacro_spc();
+    }
+    if (spc == 1)
+        --str.len; /* remove trailing space */
+    tok_str_add(&str, 0);
+#ifdef PP_DEBUG
+    printf("define %s %d: ", get_tok_str(v, NULL), t);
+    tok_print(str.str);
+#endif
+    define_push(v, t, str.str, first);
+}
+
+static inline int hash_cached_include(const char *filename)
+{
+    const unsigned char *s;
+    unsigned int h;
+
+    h = TOK_HASH_INIT;
+    s = filename;
+    while (*s) {
+        h = TOK_HASH_FUNC(h, *s);
+        s++;
+    }
+    h &= (CACHED_INCLUDES_HASH_SIZE - 1);
+    return h;
+}
+
+static CachedInclude *search_cached_include(TCCState *s1, const char *filename)
+{
+    CachedInclude *e;
+    int i, h;
+    h = hash_cached_include(filename);
+    i = s1->cached_includes_hash[h];
+    for(;;) {
+        if (i == 0)
+            break;
+        e = s1->cached_includes[i - 1];
+        if (0 == PATHCMP(e->filename, filename))
+            return e;
+        i = e->hash_next;
+    }
+    return NULL;
+}
+
+static inline void add_cached_include(TCCState *s1, const char *filename, int ifndef_macro)
+{
+    CachedInclude *e;
+    int h;
+
+    if (search_cached_include(s1, filename))
+        return;
+#ifdef INC_DEBUG
+    printf("adding cached '%s' %s\n", filename, get_tok_str(ifndef_macro, NULL));
+#endif
+    e = tcc_malloc(sizeof(CachedInclude) + strlen(filename));
+    strcpy(e->filename, filename);
+    e->ifndef_macro = ifndef_macro;
+    dynarray_add((void ***)&s1->cached_includes, &s1->nb_cached_includes, e);
+    /* add in hash table */
+    h = hash_cached_include(filename);
+    e->hash_next = s1->cached_includes_hash[h];
+    s1->cached_includes_hash[h] = s1->nb_cached_includes;
+}
+
+static void pragma_parse(TCCState *s1)
+{
+    int val;
+
+    next();
+    if (tok == TOK_pack) {
+        /*
+          This may be:
+          #pragma pack(1) // set
+          #pragma pack() // reset to default
+          #pragma pack(push,1) // push & set
+          #pragma pack(pop) // restore previous
+        */
+        next();
+        skip('(');
+        if (tok == TOK_ASM_pop) {
+            next();
+            if (s1->pack_stack_ptr <= s1->pack_stack) {
+            stk_error:
+                tcc_error("out of pack stack");
+            }
+            s1->pack_stack_ptr--;
+        } else {
+            val = 0;
+            if (tok != ')') {
+                if (tok == TOK_ASM_push) {
+                    next();
+                    if (s1->pack_stack_ptr >= s1->pack_stack + PACK_STACK_SIZE - 1)
+                        goto stk_error;
+                    s1->pack_stack_ptr++;
+                    skip(',');
+                }
+                if (tok != TOK_CINT) {
+                pack_error:
+                    tcc_error("invalid pack pragma");
+                }
+                val = tokc.i;
+                if (val < 1 || val > 16 || (val & (val - 1)) != 0)
+                    goto pack_error;
+                next();
+            }
+            *s1->pack_stack_ptr = val;
+            skip(')');
+        }
+    }
+}
+
+/* is_bof is true if first non space token at beginning of file */
+ST_FUNC void preprocess(int is_bof)
+{
+    TCCState *s1 = tcc_state;
+    int i, c, n, saved_parse_flags;
+    char buf[1024], *q;
+    Sym *s;
+
+    saved_parse_flags = parse_flags;
+    parse_flags = PARSE_FLAG_PREPROCESS | PARSE_FLAG_TOK_NUM | 
+        PARSE_FLAG_LINEFEED;
+    next_nomacro();
+ redo:
+    switch(tok) {
+    case TOK_DEFINE:
+        next_nomacro();
+        parse_define();
+        break;
+    case TOK_UNDEF:
+        next_nomacro();
+        s = define_find(tok);
+        /* undefine symbol by putting an invalid name */
+        if (s)
+            define_undef(s);
+        break;
+    case TOK_INCLUDE:
+    case TOK_INCLUDE_NEXT:
+        ch = file->buf_ptr[0];
+        /* XXX: incorrect if comments : use next_nomacro with a special mode */
+        skip_spaces();
+        if (ch == '<') {
+            c = '>';
+            goto read_name;
+        } else if (ch == '\"') {
+            c = ch;
+        read_name:
+            inp();
+            q = buf;
+            while (ch != c && ch != '\n' && ch != CH_EOF) {
+                if ((q - buf) < sizeof(buf) - 1)
+                    *q++ = ch;
+                if (ch == '\\') {
+                    if (handle_stray_noerror() == 0)
+                        --q;
+                } else
+                    inp();
+            }
+            *q = '\0';
+            minp();
+#if 0
+            /* eat all spaces and comments after include */
+            /* XXX: slightly incorrect */
+            while (ch1 != '\n' && ch1 != CH_EOF)
+                inp();
+#endif
+        } else {
+            /* computed #include : either we have only strings or
+               we have anything enclosed in '<>' */
+            next();
+            buf[0] = '\0';
+            if (tok == TOK_STR) {
+                while (tok != TOK_LINEFEED) {
+                    if (tok != TOK_STR) {
+                    include_syntax:
+                        tcc_error("'#include' expects \"FILENAME\" or <FILENAME>");
+                    }
+                    pstrcat(buf, sizeof(buf), (char *)tokc.cstr->data);
+                    next();
+                }
+                c = '\"';
+            } else {
+                int len;
+                while (tok != TOK_LINEFEED) {
+                    pstrcat(buf, sizeof(buf), get_tok_str(tok, &tokc));
+                    next();
+                }
+                len = strlen(buf);
+                /* check syntax and remove '<>' */
+                if (len < 2 || buf[0] != '<' || buf[len - 1] != '>')
+                    goto include_syntax;
+                memmove(buf, buf + 1, len - 2);
+                buf[len - 2] = '\0';
+                c = '>';
+            }
+        }
+
+        if (s1->include_stack_ptr >= s1->include_stack + INCLUDE_STACK_SIZE)
+            tcc_error("#include recursion too deep");
+        /* store current file in stack, but increment stack later below */
+        *s1->include_stack_ptr = file;
+
+        n = s1->nb_include_paths + s1->nb_sysinclude_paths;
+        for (i = -2; i < n; ++i) {
+            char buf1[sizeof file->filename];
+            CachedInclude *e;
+            BufferedFile **f;
+            const char *path;
+
+            if (i == -2) {
+                /* check absolute include path */
+                if (!IS_ABSPATH(buf))
+                    continue;
+                buf1[0] = 0;
+                i = n; /* force end loop */
+
+            } else if (i == -1) {
+                /* search in current dir if "header.h" */
+                if (c != '\"')
+                    continue;
+                path = file->filename;
+                pstrncpy(buf1, path, tcc_basename(path) - path);
+
+            } else {
+                /* search in all the include paths */
+                if (i < s1->nb_include_paths)
+                    path = s1->include_paths[i];
+                else
+                    path = s1->sysinclude_paths[i - s1->nb_include_paths];
+                pstrcpy(buf1, sizeof(buf1), path);
+                pstrcat(buf1, sizeof(buf1), "/");
+            }
+
+            pstrcat(buf1, sizeof(buf1), buf);
+
+            if (tok == TOK_INCLUDE_NEXT)
+                for (f = s1->include_stack_ptr; f >= s1->include_stack; --f)
+                    if (0 == PATHCMP((*f)->filename, buf1)) {
+#ifdef INC_DEBUG
+                        printf("%s: #include_next skipping %s\n", file->filename, buf1);
+#endif
+                        goto include_trynext;
+                    }
+
+            e = search_cached_include(s1, buf1);
+            if (e && define_find(e->ifndef_macro)) {
+                /* no need to parse the include because the 'ifndef macro'
+                   is defined */
+#ifdef INC_DEBUG
+                printf("%s: skipping cached %s\n", file->filename, buf1);
+#endif
+                goto include_done;
+            }
+
+            if (tcc_open(s1, buf1) < 0)
+include_trynext:
+                continue;
+
+#ifdef INC_DEBUG
+            printf("%s: including %s\n", file->prev->filename, file->filename);
+#endif
+            /* update target deps */
+            dynarray_add((void ***)&s1->target_deps, &s1->nb_target_deps,
+                    tcc_strdup(buf1));
+            /* push current file in stack */
+            ++s1->include_stack_ptr;
+            /* add include file debug info */
+            if (s1->do_debug)
+                put_stabs(file->filename, N_BINCL, 0, 0, 0);
+            tok_flags |= TOK_FLAG_BOF | TOK_FLAG_BOL;
+            ch = file->buf_ptr[0];
+            goto the_end;
+        }
+        tcc_error("include file '%s' not found", buf);
+include_done:
+        break;
+    case TOK_IFNDEF:
+        c = 1;
+        goto do_ifdef;
+    case TOK_IF:
+        c = expr_preprocess();
+        goto do_if;
+    case TOK_IFDEF:
+        c = 0;
+    do_ifdef:
+        next_nomacro();
+        if (tok < TOK_IDENT)
+            tcc_error("invalid argument for '#if%sdef'", c ? "n" : "");
+        if (is_bof) {
+            if (c) {
+#ifdef INC_DEBUG
+                printf("#ifndef %s\n", get_tok_str(tok, NULL));
+#endif
+                file->ifndef_macro = tok;
+            }
+        }
+        c = (define_find(tok) != 0) ^ c;
+    do_if:
+        if (s1->ifdef_stack_ptr >= s1->ifdef_stack + IFDEF_STACK_SIZE)
+            tcc_error("memory full");
+        *s1->ifdef_stack_ptr++ = c;
+        goto test_skip;
+    case TOK_ELSE:
+        if (s1->ifdef_stack_ptr == s1->ifdef_stack)
+            tcc_error("#else without matching #if");
+        if (s1->ifdef_stack_ptr[-1] & 2)
+            tcc_error("#else after #else");
+        c = (s1->ifdef_stack_ptr[-1] ^= 3);
+        goto test_else;
+    case TOK_ELIF:
+        if (s1->ifdef_stack_ptr == s1->ifdef_stack)
+            tcc_error("#elif without matching #if");
+        c = s1->ifdef_stack_ptr[-1];
+        if (c > 1)
+            tcc_error("#elif after #else");
+        /* last #if/#elif expression was true: we skip */
+        if (c == 1)
+            goto skip;
+        c = expr_preprocess();
+        s1->ifdef_stack_ptr[-1] = c;
+    test_else:
+        if (s1->ifdef_stack_ptr == file->ifdef_stack_ptr + 1)
+            file->ifndef_macro = 0;
+    test_skip:
+        if (!(c & 1)) {
+        skip:
+            preprocess_skip();
+            is_bof = 0;
+            goto redo;
+        }
+        break;
+    case TOK_ENDIF:
+        if (s1->ifdef_stack_ptr <= file->ifdef_stack_ptr)
+            tcc_error("#endif without matching #if");
+        s1->ifdef_stack_ptr--;
+        /* '#ifndef macro' was at the start of file. Now we check if
+           an '#endif' is exactly at the end of file */
+        if (file->ifndef_macro &&
+            s1->ifdef_stack_ptr == file->ifdef_stack_ptr) {
+            file->ifndef_macro_saved = file->ifndef_macro;
+            /* need to set to zero to avoid false matches if another
+               #ifndef at middle of file */
+            file->ifndef_macro = 0;
+            while (tok != TOK_LINEFEED)
+                next_nomacro();
+            tok_flags |= TOK_FLAG_ENDIF;
+            goto the_end;
+        }
+        break;
+    case TOK_LINE:
+        next();
+        if (tok != TOK_CINT)
+            tcc_error("#line");
+        file->line_num = tokc.i - 1; /* the line number will be incremented after */
+        next();
+        if (tok != TOK_LINEFEED) {
+            if (tok != TOK_STR)
+                tcc_error("#line");
+            pstrcpy(file->filename, sizeof(file->filename), 
+                    (char *)tokc.cstr->data);
+        }
+        break;
+    case TOK_ERROR:
+    case TOK_WARNING:
+        c = tok;
+        ch = file->buf_ptr[0];
+        skip_spaces();
+        q = buf;
+        while (ch != '\n' && ch != CH_EOF) {
+            if ((q - buf) < sizeof(buf) - 1)
+                *q++ = ch;
+            if (ch == '\\') {
+                if (handle_stray_noerror() == 0)
+                    --q;
+            } else
+                inp();
+        }
+        *q = '\0';
+        if (c == TOK_ERROR)
+            tcc_error("#error %s", buf);
+        else
+            tcc_warning("#warning %s", buf);
+        break;
+    case TOK_PRAGMA:
+        pragma_parse(s1);
+        break;
+    default:
+        if (tok == TOK_LINEFEED || tok == '!' || tok == TOK_PPNUM) {
+            /* '!' is ignored to allow C scripts. numbers are ignored
+               to emulate cpp behaviour */
+        } else {
+            if (!(saved_parse_flags & PARSE_FLAG_ASM_COMMENTS))
+                tcc_warning("Ignoring unknown preprocessing directive #%s", get_tok_str(tok, &tokc));
+            else {
+                /* this is a gas line comment in an 'S' file. */
+                file->buf_ptr = parse_line_comment(file->buf_ptr);
+                goto the_end;
+            }
+        }
+        break;
+    }
+    /* ignore other preprocess commands or #! for C scripts */
+    while (tok != TOK_LINEFEED)
+        next_nomacro();
+ the_end:
+    parse_flags = saved_parse_flags;
+}
+
+/* evaluate escape codes in a string. */
+static void parse_escape_string(CString *outstr, const uint8_t *buf, int is_long)
+{
+    int c, n;
+    const uint8_t *p;
+
+    p = buf;
+    for(;;) {
+        c = *p;
+        if (c == '\0')
+            break;
+        if (c == '\\') {
+            p++;
+            /* escape */
+            c = *p;
+            switch(c) {
+            case '0': case '1': case '2': case '3':
+            case '4': case '5': case '6': case '7':
+                /* at most three octal digits */
+                n = c - '0';
+                p++;
+                c = *p;
+                if (isoct(c)) {
+                    n = n * 8 + c - '0';
+                    p++;
+                    c = *p;
+                    if (isoct(c)) {
+                        n = n * 8 + c - '0';
+                        p++;
+                    }
+                }
+                c = n;
+                goto add_char_nonext;
+            case 'x':
+            case 'u':
+            case 'U':
+                p++;
+                n = 0;
+                for(;;) {
+                    c = *p;
+                    if (c >= 'a' && c <= 'f')
+                        c = c - 'a' + 10;
+                    else if (c >= 'A' && c <= 'F')
+                        c = c - 'A' + 10;
+                    else if (isnum(c))
+                        c = c - '0';
+                    else
+                        break;
+                    n = n * 16 + c;
+                    p++;
+                }
+                c = n;
+                goto add_char_nonext;
+            case 'a':
+                c = '\a';
+                break;
+            case 'b':
+                c = '\b';
+                break;
+            case 'f':
+                c = '\f';
+                break;
+            case 'n':
+                c = '\n';
+                break;
+            case 'r':
+                c = '\r';
+                break;
+            case 't':
+                c = '\t';
+                break;
+            case 'v':
+                c = '\v';
+                break;
+            case 'e':
+                if (!gnu_ext)
+                    goto invalid_escape;
+                c = 27;
+                break;
+            case '\'':
+            case '\"':
+            case '\\': 
+            case '?':
+                break;
+            default:
+            invalid_escape:
+                if (c >= '!' && c <= '~')
+                    tcc_warning("unknown escape sequence: \'\\%c\'", c);
+                else
+                    tcc_warning("unknown escape sequence: \'\\x%x\'", c);
+                break;
+            }
+        }
+        p++;
+    add_char_nonext:
+        if (!is_long)
+            cstr_ccat(outstr, c);
+        else
+            cstr_wccat(outstr, c);
+    }
+    /* add a trailing '\0' */
+    if (!is_long)
+        cstr_ccat(outstr, '\0');
+    else
+        cstr_wccat(outstr, '\0');
+}
+
+/* we use 64 bit numbers */
+#define BN_SIZE 2
+
+/* bn = (bn << shift) | or_val */
+static void bn_lshift(unsigned int *bn, int shift, int or_val)
+{
+    int i;
+    unsigned int v;
+    for(i=0;i<BN_SIZE;i++) {
+        v = bn[i];
+        bn[i] = (v << shift) | or_val;
+        or_val = v >> (32 - shift);
+    }
+}
+
+static void bn_zero(unsigned int *bn)
+{
+    int i;
+    for(i=0;i<BN_SIZE;i++) {
+        bn[i] = 0;
+    }
+}
+
+/* parse number in null terminated string 'p' and return it in the
+   current token */
+static void parse_number(const char *p)
+{
+    int b, t, shift, frac_bits, s, exp_val, ch;
+    char *q;
+    unsigned int bn[BN_SIZE];
+    double d;
+
+    /* number */
+    q = token_buf;
+    ch = *p++;
+    t = ch;
+    ch = *p++;
+    *q++ = t;
+    b = 10;
+    if (t == '.') {
+        goto float_frac_parse;
+    } else if (t == '0') {
+        if (ch == 'x' || ch == 'X') {
+            q--;
+            ch = *p++;
+            b = 16;
+        } else if (tcc_ext && (ch == 'b' || ch == 'B')) {
+            q--;
+            ch = *p++;
+            b = 2;
+        }
+    }
+    /* parse all digits. cannot check octal numbers at this stage
+       because of floating point constants */
+    while (1) {
+        if (ch >= 'a' && ch <= 'f')
+            t = ch - 'a' + 10;
+        else if (ch >= 'A' && ch <= 'F')
+            t = ch - 'A' + 10;
+        else if (isnum(ch))
+            t = ch - '0';
+        else
+            break;
+        if (t >= b)
+            break;
+        if (q >= token_buf + STRING_MAX_SIZE) {
+        num_too_long:
+            tcc_error("number too long");
+        }
+        *q++ = ch;
+        ch = *p++;
+    }
+    if (ch == '.' ||
+        ((ch == 'e' || ch == 'E') && b == 10) ||
+        ((ch == 'p' || ch == 'P') && (b == 16 || b == 2))) {
+        if (b != 10) {
+            /* NOTE: strtox should support that for hexa numbers, but
+               non ISOC99 libcs do not support it, so we prefer to do
+               it by hand */
+            /* hexadecimal or binary floats */
+            /* XXX: handle overflows */
+            *q = '\0';
+            if (b == 16)
+                shift = 4;
+            else 
+                shift = 2;
+            bn_zero(bn);
+            q = token_buf;
+            while (1) {
+                t = *q++;
+                if (t == '\0') {
+                    break;
+                } else if (t >= 'a') {
+                    t = t - 'a' + 10;
+                } else if (t >= 'A') {
+                    t = t - 'A' + 10;
+                } else {
+                    t = t - '0';
+                }
+                bn_lshift(bn, shift, t);
+            }
+            frac_bits = 0;
+            if (ch == '.') {
+                ch = *p++;
+                while (1) {
+                    t = ch;
+                    if (t >= 'a' && t <= 'f') {
+                        t = t - 'a' + 10;
+                    } else if (t >= 'A' && t <= 'F') {
+                        t = t - 'A' + 10;
+                    } else if (t >= '0' && t <= '9') {
+                        t = t - '0';
+                    } else {
+                        break;
+                    }
+                    if (t >= b)
+                        tcc_error("invalid digit");
+                    bn_lshift(bn, shift, t);
+                    frac_bits += shift;
+                    ch = *p++;
+                }
+            }
+            if (ch != 'p' && ch != 'P')
+                expect("exponent");
+            ch = *p++;
+            s = 1;
+            exp_val = 0;
+            if (ch == '+') {
+                ch = *p++;
+            } else if (ch == '-') {
+                s = -1;
+                ch = *p++;
+            }
+            if (ch < '0' || ch > '9')
+                expect("exponent digits");
+            while (ch >= '0' && ch <= '9') {
+                exp_val = exp_val * 10 + ch - '0';
+                ch = *p++;
+            }
+            exp_val = exp_val * s;
+            
+            /* now we can generate the number */
+            /* XXX: should patch directly float number */
+            d = (double)bn[1] * 4294967296.0 + (double)bn[0];
+            d = ldexp(d, exp_val - frac_bits);
+            t = toup(ch);
+            if (t == 'F') {
+                ch = *p++;
+                tok = TOK_CFLOAT;
+                /* float : should handle overflow */
+                tokc.f = (float)d;
+            } else if (t == 'L') {
+                ch = *p++;
+#ifdef TCC_TARGET_PE
+                tok = TOK_CDOUBLE;
+                tokc.d = d;
+#else
+                tok = TOK_CLDOUBLE;
+                /* XXX: not large enough */
+                tokc.ld = (long double)d;
+#endif
+            } else {
+                tok = TOK_CDOUBLE;
+                tokc.d = d;
+            }
+        } else {
+            /* decimal floats */
+            if (ch == '.') {
+                if (q >= token_buf + STRING_MAX_SIZE)
+                    goto num_too_long;
+                *q++ = ch;
+                ch = *p++;
+            float_frac_parse:
+                while (ch >= '0' && ch <= '9') {
+                    if (q >= token_buf + STRING_MAX_SIZE)
+                        goto num_too_long;
+                    *q++ = ch;
+                    ch = *p++;
+                }
+            }
+            if (ch == 'e' || ch == 'E') {
+                if (q >= token_buf + STRING_MAX_SIZE)
+                    goto num_too_long;
+                *q++ = ch;
+                ch = *p++;
+                if (ch == '-' || ch == '+') {
+                    if (q >= token_buf + STRING_MAX_SIZE)
+                        goto num_too_long;
+                    *q++ = ch;
+                    ch = *p++;
+                }
+                if (ch < '0' || ch > '9')
+                    expect("exponent digits");
+                while (ch >= '0' && ch <= '9') {
+                    if (q >= token_buf + STRING_MAX_SIZE)
+                        goto num_too_long;
+                    *q++ = ch;
+                    ch = *p++;
+                }
+            }
+            *q = '\0';
+            t = toup(ch);
+            errno = 0;
+            if (t == 'F') {
+                ch = *p++;
+                tok = TOK_CFLOAT;
+                tokc.f = strtof(token_buf, NULL);
+            } else if (t == 'L') {
+                ch = *p++;
+#ifdef TCC_TARGET_PE
+                tok = TOK_CDOUBLE;
+                tokc.d = strtod(token_buf, NULL);
+#else
+                tok = TOK_CLDOUBLE;
+                tokc.ld = strtold(token_buf, NULL);
+#endif
+            } else {
+                tok = TOK_CDOUBLE;
+                tokc.d = strtod(token_buf, NULL);
+            }
+        }
+    } else {
+        unsigned long long n, n1;
+        int lcount, ucount;
+
+        /* integer number */
+        *q = '\0';
+        q = token_buf;
+        if (b == 10 && *q == '0') {
+            b = 8;
+            q++;
+        }
+        n = 0;
+        while(1) {
+            t = *q++;
+            /* no need for checks except for base 10 / 8 errors */
+            if (t == '\0') {
+                break;
+            } else if (t >= 'a') {
+                t = t - 'a' + 10;
+            } else if (t >= 'A') {
+                t = t - 'A' + 10;
+            } else {
+                t = t - '0';
+                if (t >= b)
+                    tcc_error("invalid digit");
+            }
+            n1 = n;
+            n = n * b + t;
+            /* detect overflow */
+            /* XXX: this test is not reliable */
+            if (n < n1)
+                tcc_error("integer constant overflow");
+        }
+        
+        /* XXX: not exactly ANSI compliant */
+        if ((n & 0xffffffff00000000LL) != 0) {
+            if ((n >> 63) != 0)
+                tok = TOK_CULLONG;
+            else
+                tok = TOK_CLLONG;
+        } else if (n > 0x7fffffff) {
+            tok = TOK_CUINT;
+        } else {
+            tok = TOK_CINT;
+        }
+        lcount = 0;
+        ucount = 0;
+        for(;;) {
+            t = toup(ch);
+            if (t == 'L') {
+                if (lcount >= 2)
+                    tcc_error("three 'l's in integer constant");
+                lcount++;
+#if !defined TCC_TARGET_X86_64 || defined TCC_TARGET_PE
+                if (lcount == 2) {
+#endif
+                    if (tok == TOK_CINT)
+                        tok = TOK_CLLONG;
+                    else if (tok == TOK_CUINT)
+                        tok = TOK_CULLONG;
+#if !defined TCC_TARGET_X86_64 || defined TCC_TARGET_PE
+                }
+#endif
+                ch = *p++;
+            } else if (t == 'U') {
+                if (ucount >= 1)
+                    tcc_error("two 'u's in integer constant");
+                ucount++;
+                if (tok == TOK_CINT)
+                    tok = TOK_CUINT;
+                else if (tok == TOK_CLLONG)
+                    tok = TOK_CULLONG;
+                ch = *p++;
+            } else {
+                break;
+            }
+        }
+        if (tok == TOK_CINT || tok == TOK_CUINT)
+            tokc.ui = n;
+        else
+            tokc.ull = n;
+    }
+    if (ch)
+        tcc_error("invalid number\n");
+}
+
+
+#define PARSE2(c1, tok1, c2, tok2)              \
+    case c1:                                    \
+        PEEKC(c, p);                            \
+        if (c == c2) {                          \
+            p++;                                \
+            tok = tok2;                         \
+        } else {                                \
+            tok = tok1;                         \
+        }                                       \
+        break;
+
+/* return next token without macro substitution */
+static inline void next_nomacro1(void)
+{
+    int t, c, is_long;
+    TokenSym *ts;
+    uint8_t *p, *p1;
+    unsigned int h;
+
+    p = file->buf_ptr;
+ redo_no_start:
+    c = *p;
+    switch(c) {
+    case ' ':
+    case '\t':
+        tok = c;
+        p++;
+        goto keep_tok_flags;
+    case '\f':
+    case '\v':
+    case '\r':
+        p++;
+        goto redo_no_start;
+    case '\\':
+        /* first look if it is in fact an end of buffer */
+        if (p >= file->buf_end) {
+            file->buf_ptr = p;
+            handle_eob();
+            p = file->buf_ptr;
+            if (p >= file->buf_end)
+                goto parse_eof;
+            else
+                goto redo_no_start;
+        } else {
+            file->buf_ptr = p;
+            ch = *p;
+            handle_stray();
+            p = file->buf_ptr;
+            goto redo_no_start;
+        }
+    parse_eof:
+        {
+            TCCState *s1 = tcc_state;
+            if ((parse_flags & PARSE_FLAG_LINEFEED)
+                && !(tok_flags & TOK_FLAG_EOF)) {
+                tok_flags |= TOK_FLAG_EOF;
+                tok = TOK_LINEFEED;
+                goto keep_tok_flags;
+            } else if (!(parse_flags & PARSE_FLAG_PREPROCESS)) {
+                tok = TOK_EOF;
+            } else if (s1->ifdef_stack_ptr != file->ifdef_stack_ptr) {
+                tcc_error("missing #endif");
+            } else if (s1->include_stack_ptr == s1->include_stack) {
+                /* no include left : end of file. */
+                tok = TOK_EOF;
+            } else {
+                tok_flags &= ~TOK_FLAG_EOF;
+                /* pop include file */
+                
+                /* test if previous '#endif' was after a #ifdef at
+                   start of file */
+                if (tok_flags & TOK_FLAG_ENDIF) {
+#ifdef INC_DEBUG
+                    printf("#endif %s\n", get_tok_str(file->ifndef_macro_saved, NULL));
+#endif
+                    add_cached_include(s1, file->filename, file->ifndef_macro_saved);
+                    tok_flags &= ~TOK_FLAG_ENDIF;
+                }
+
+                /* add end of include file debug info */
+                if (tcc_state->do_debug) {
+                    put_stabd(N_EINCL, 0, 0);
+                }
+                /* pop include stack */
+                tcc_close();
+                s1->include_stack_ptr--;
+                p = file->buf_ptr;
+                goto redo_no_start;
+            }
+        }
+        break;
+
+    case '\n':
+        file->line_num++;
+        tok_flags |= TOK_FLAG_BOL;
+        p++;
+maybe_newline:
+        if (0 == (parse_flags & PARSE_FLAG_LINEFEED))
+            goto redo_no_start;
+        tok = TOK_LINEFEED;
+        goto keep_tok_flags;
+
+    case '#':
+        /* XXX: simplify */
+        PEEKC(c, p);
+        if ((tok_flags & TOK_FLAG_BOL) && 
+            (parse_flags & PARSE_FLAG_PREPROCESS)) {
+            file->buf_ptr = p;
+            preprocess(tok_flags & TOK_FLAG_BOF);
+            p = file->buf_ptr;
+            goto maybe_newline;
+        } else {
+            if (c == '#') {
+                p++;
+                tok = TOK_TWOSHARPS;
+            } else {
+                if (parse_flags & PARSE_FLAG_ASM_COMMENTS) {
+                    p = parse_line_comment(p - 1);
+                    goto redo_no_start;
+                } else {
+                    tok = '#';
+                }
+            }
+        }
+        break;
+
+    case 'a': case 'b': case 'c': case 'd':
+    case 'e': case 'f': case 'g': case 'h':
+    case 'i': case 'j': case 'k': case 'l':
+    case 'm': case 'n': case 'o': case 'p':
+    case 'q': case 'r': case 's': case 't':
+    case 'u': case 'v': case 'w': case 'x':
+    case 'y': case 'z': 
+    case 'A': case 'B': case 'C': case 'D':
+    case 'E': case 'F': case 'G': case 'H':
+    case 'I': case 'J': case 'K': 
+    case 'M': case 'N': case 'O': case 'P':
+    case 'Q': case 'R': case 'S': case 'T':
+    case 'U': case 'V': case 'W': case 'X':
+    case 'Y': case 'Z': 
+    case '_':
+    parse_ident_fast:
+        p1 = p;
+        h = TOK_HASH_INIT;
+        h = TOK_HASH_FUNC(h, c);
+        p++;
+        for(;;) {
+            c = *p;
+            if (!isidnum_table[c-CH_EOF])
+                break;
+            h = TOK_HASH_FUNC(h, c);
+            p++;
+        }
+        if (c != '\\') {
+            TokenSym **pts;
+            int len;
+
+            /* fast case : no stray found, so we have the full token
+               and we have already hashed it */
+            len = p - p1;
+            h &= (TOK_HASH_SIZE - 1);
+            pts = &hash_ident[h];
+            for(;;) {
+                ts = *pts;
+                if (!ts)
+                    break;
+                if (ts->len == len && !memcmp(ts->str, p1, len))
+                    goto token_found;
+                pts = &(ts->hash_next);
+            }
+            ts = tok_alloc_new(pts, p1, len);
+        token_found: ;
+        } else {
+            /* slower case */
+            cstr_reset(&tokcstr);
+
+            while (p1 < p) {
+                cstr_ccat(&tokcstr, *p1);
+                p1++;
+            }
+            p--;
+            PEEKC(c, p);
+        parse_ident_slow:
+            while (isidnum_table[c-CH_EOF]) {
+                cstr_ccat(&tokcstr, c);
+                PEEKC(c, p);
+            }
+            ts = tok_alloc(tokcstr.data, tokcstr.size);
+        }
+        tok = ts->tok;
+        break;
+    case 'L':
+        t = p[1];
+        if (t != '\\' && t != '\'' && t != '\"') {
+            /* fast case */
+            goto parse_ident_fast;
+        } else {
+            PEEKC(c, p);
+            if (c == '\'' || c == '\"') {
+                is_long = 1;
+                goto str_const;
+            } else {
+                cstr_reset(&tokcstr);
+                cstr_ccat(&tokcstr, 'L');
+                goto parse_ident_slow;
+            }
+        }
+        break;
+    case '0': case '1': case '2': case '3':
+    case '4': case '5': case '6': case '7':
+    case '8': case '9':
+
+        cstr_reset(&tokcstr);
+        /* after the first digit, accept digits, alpha, '.' or sign if
+           prefixed by 'eEpP' */
+    parse_num:
+        for(;;) {
+            t = c;
+            cstr_ccat(&tokcstr, c);
+            PEEKC(c, p);
+            if (!(isnum(c) || isid(c) || c == '.' ||
+                  ((c == '+' || c == '-') && 
+                   (t == 'e' || t == 'E' || t == 'p' || t == 'P'))))
+                break;
+        }
+        /* We add a trailing '\0' to ease parsing */
+        cstr_ccat(&tokcstr, '\0');
+        tokc.cstr = &tokcstr;
+        tok = TOK_PPNUM;
+        break;
+    case '.':
+        /* special dot handling because it can also start a number */
+        PEEKC(c, p);
+        if (isnum(c)) {
+            cstr_reset(&tokcstr);
+            cstr_ccat(&tokcstr, '.');
+            goto parse_num;
+        } else if (c == '.') {
+            PEEKC(c, p);
+            if (c != '.')
+                expect("'.'");
+            PEEKC(c, p);
+            tok = TOK_DOTS;
+        } else {
+            tok = '.';
+        }
+        break;
+    case '\'':
+    case '\"':
+        is_long = 0;
+    str_const:
+        {
+            CString str;
+            int sep;
+
+            sep = c;
+
+            /* parse the string */
+            cstr_new(&str);
+            p = parse_pp_string(p, sep, &str);
+            cstr_ccat(&str, '\0');
+            
+            /* eval the escape (should be done as TOK_PPNUM) */
+            cstr_reset(&tokcstr);
+            parse_escape_string(&tokcstr, str.data, is_long);
+            cstr_free(&str);
+
+            if (sep == '\'') {
+                int char_size;
+                /* XXX: make it portable */
+                if (!is_long)
+                    char_size = 1;
+                else
+                    char_size = sizeof(nwchar_t);
+                if (tokcstr.size <= char_size)
+                    tcc_error("empty character constant");
+                if (tokcstr.size > 2 * char_size)
+                    tcc_warning("multi-character character constant");
+                if (!is_long) {
+                    tokc.i = *(int8_t *)tokcstr.data;
+                    tok = TOK_CCHAR;
+                } else {
+                    tokc.i = *(nwchar_t *)tokcstr.data;
+                    tok = TOK_LCHAR;
+                }
+            } else {
+                tokc.cstr = &tokcstr;
+                if (!is_long)
+                    tok = TOK_STR;
+                else
+                    tok = TOK_LSTR;
+            }
+        }
+        break;
+
+    case '<':
+        PEEKC(c, p);
+        if (c == '=') {
+            p++;
+            tok = TOK_LE;
+        } else if (c == '<') {
+            PEEKC(c, p);
+            if (c == '=') {
+                p++;
+                tok = TOK_A_SHL;
+            } else {
+                tok = TOK_SHL;
+            }
+        } else {
+            tok = TOK_LT;
+        }
+        break;
+        
+    case '>':
+        PEEKC(c, p);
+        if (c == '=') {
+            p++;
+            tok = TOK_GE;
+        } else if (c == '>') {
+            PEEKC(c, p);
+            if (c == '=') {
+                p++;
+                tok = TOK_A_SAR;
+            } else {
+                tok = TOK_SAR;
+            }
+        } else {
+            tok = TOK_GT;
+        }
+        break;
+        
+    case '&':
+        PEEKC(c, p);
+        if (c == '&') {
+            p++;
+            tok = TOK_LAND;
+        } else if (c == '=') {
+            p++;
+            tok = TOK_A_AND;
+        } else {
+            tok = '&';
+        }
+        break;
+        
+    case '|':
+        PEEKC(c, p);
+        if (c == '|') {
+            p++;
+            tok = TOK_LOR;
+        } else if (c == '=') {
+            p++;
+            tok = TOK_A_OR;
+        } else {
+            tok = '|';
+        }
+        break;
+
+    case '+':
+        PEEKC(c, p);
+        if (c == '+') {
+            p++;
+            tok = TOK_INC;
+        } else if (c == '=') {
+            p++;
+            tok = TOK_A_ADD;
+        } else {
+            tok = '+';
+        }
+        break;
+        
+    case '-':
+        PEEKC(c, p);
+        if (c == '-') {
+            p++;
+            tok = TOK_DEC;
+        } else if (c == '=') {
+            p++;
+            tok = TOK_A_SUB;
+        } else if (c == '>') {
+            p++;
+            tok = TOK_ARROW;
+        } else {
+            tok = '-';
+        }
+        break;
+
+    PARSE2('!', '!', '=', TOK_NE)
+    PARSE2('=', '=', '=', TOK_EQ)
+    PARSE2('*', '*', '=', TOK_A_MUL)
+    PARSE2('%', '%', '=', TOK_A_MOD)
+    PARSE2('^', '^', '=', TOK_A_XOR)
+        
+        /* comments or operator */
+    case '/':
+        PEEKC(c, p);
+        if (c == '*') {
+            p = parse_comment(p);
+            /* comments replaced by a blank */
+            tok = ' ';
+            goto keep_tok_flags;
+        } else if (c == '/') {
+            p = parse_line_comment(p);
+            tok = ' ';
+            goto keep_tok_flags;
+        } else if (c == '=') {
+            p++;
+            tok = TOK_A_DIV;
+        } else {
+            tok = '/';
+        }
+        break;
+        
+        /* simple tokens */
+    case '(':
+    case ')':
+    case '[':
+    case ']':
+    case '{':
+    case '}':
+    case ',':
+    case ';':
+    case ':':
+    case '?':
+    case '~':
+    case '$': /* only used in assembler */
+    case '@': /* dito */
+        tok = c;
+        p++;
+        break;
+    default:
+        tcc_error("unrecognized character \\x%02x", c);
+        break;
+    }
+    tok_flags = 0;
+keep_tok_flags:
+    file->buf_ptr = p;
+#if defined(PARSE_DEBUG)
+    printf("token = %s\n", get_tok_str(tok, &tokc));
+#endif
+}
+
+/* return next token without macro substitution. Can read input from
+   macro_ptr buffer */
+static void next_nomacro_spc(void)
+{
+    if (macro_ptr) {
+    redo:
+        tok = *macro_ptr;
+        if (tok) {
+            TOK_GET(&tok, &macro_ptr, &tokc);
+            if (tok == TOK_LINENUM) {
+                file->line_num = tokc.i;
+                goto redo;
+            }
+        }
+    } else {
+        next_nomacro1();
+    }
+}
+
+ST_FUNC void next_nomacro(void)
+{
+    do {
+        next_nomacro_spc();
+    } while (is_space(tok));
+}
+ 
+/* substitute args in macro_str and return allocated string */
+static int *macro_arg_subst(Sym **nested_list, const int *macro_str, Sym *args)
+{
+    int last_tok, t, spc;
+    const int *st;
+    Sym *s;
+    CValue cval;
+    TokenString str;
+    CString cstr;
+
+    tok_str_new(&str);
+    last_tok = 0;
+    while(1) {
+        TOK_GET(&t, &macro_str, &cval);
+        if (!t)
+            break;
+        if (t == '#') {
+            /* stringize */
+            TOK_GET(&t, &macro_str, &cval);
+            if (!t)
+                break;
+            s = sym_find2(args, t);
+            if (s) {
+                cstr_new(&cstr);
+                st = s->d;
+                spc = 0;
+                while (*st) {
+                    TOK_GET(&t, &st, &cval);
+                    if (!check_space(t, &spc))
+                        cstr_cat(&cstr, get_tok_str(t, &cval));
+                }
+                cstr.size -= spc;
+                cstr_ccat(&cstr, '\0');
+#ifdef PP_DEBUG
+                printf("stringize: %s\n", (char *)cstr.data);
+#endif
+                /* add string */
+                cval.cstr = &cstr;
+                tok_str_add2(&str, TOK_STR, &cval);
+                cstr_free(&cstr);
+            } else {
+                tok_str_add2(&str, t, &cval);
+            }
+        } else if (t >= TOK_IDENT) {
+            s = sym_find2(args, t);
+            if (s) {
+                st = s->d;
+                /* if '##' is present before or after, no arg substitution */
+                if (*macro_str == TOK_TWOSHARPS || last_tok == TOK_TWOSHARPS) {
+                    /* special case for var arg macros : ## eats the
+                       ',' if empty VA_ARGS variable. */
+                    /* XXX: test of the ',' is not 100%
+                       reliable. should fix it to avoid security
+                       problems */
+                    if (gnu_ext && s->type.t &&
+                        last_tok == TOK_TWOSHARPS && 
+                        str.len >= 2 && str.str[str.len - 2] == ',') {
+                        if (*st == 0) {
+                            /* suppress ',' '##' */
+                            str.len -= 2;
+                        } else {
+                            /* suppress '##' and add variable */
+                            str.len--;
+                            goto add_var;
+                        }
+                    } else {
+                        int t1;
+                    add_var:
+                        for(;;) {
+                            TOK_GET(&t1, &st, &cval);
+                            if (!t1)
+                                break;
+                            tok_str_add2(&str, t1, &cval);
+                        }
+                    }
+                } else {
+                    /* NOTE: the stream cannot be read when macro
+                       substituing an argument */
+                    macro_subst(&str, nested_list, st, NULL);
+                }
+            } else {
+                tok_str_add(&str, t);
+            }
+        } else {
+            tok_str_add2(&str, t, &cval);
+        }
+        last_tok = t;
+    }
+    tok_str_add(&str, 0);
+    return str.str;
+}
+
+static char const ab_month_name[12][4] =
+{
+    "Jan", "Feb", "Mar", "Apr", "May", "Jun",
+    "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"
+};
+
+/* do macro substitution of current token with macro 's' and add
+   result to (tok_str,tok_len). 'nested_list' is the list of all
+   macros we got inside to avoid recursing. Return non zero if no
+   substitution needs to be done */
+static int macro_subst_tok(TokenString *tok_str,
+                           Sym **nested_list, Sym *s, struct macro_level **can_read_stream)
+{
+    Sym *args, *sa, *sa1;
+    int mstr_allocated, parlevel, *mstr, t, t1, spc;
+    const int *p;
+    TokenString str;
+    char *cstrval;
+    CValue cval;
+    CString cstr;
+    char buf[32];
+    
+    /* if symbol is a macro, prepare substitution */
+    /* special macros */
+    if (tok == TOK___LINE__) {
+        snprintf(buf, sizeof(buf), "%d", file->line_num);
+        cstrval = buf;
+        t1 = TOK_PPNUM;
+        goto add_cstr1;
+    } else if (tok == TOK___FILE__) {
+        cstrval = file->filename;
+        goto add_cstr;
+    } else if (tok == TOK___DATE__ || tok == TOK___TIME__) {
+        time_t ti;
+        struct tm *tm;
+
+        time(&ti);
+        tm = localtime(&ti);
+        if (tok == TOK___DATE__) {
+            snprintf(buf, sizeof(buf), "%s %2d %d", 
+                     ab_month_name[tm->tm_mon], tm->tm_mday, tm->tm_year + 1900);
+        } else {
+            snprintf(buf, sizeof(buf), "%02d:%02d:%02d", 
+                     tm->tm_hour, tm->tm_min, tm->tm_sec);
+        }
+        cstrval = buf;
+    add_cstr:
+        t1 = TOK_STR;
+    add_cstr1:
+        cstr_new(&cstr);
+        cstr_cat(&cstr, cstrval);
+        cstr_ccat(&cstr, '\0');
+        cval.cstr = &cstr;
+        tok_str_add2(tok_str, t1, &cval);
+        cstr_free(&cstr);
+    } else {
+        mstr = s->d;
+        mstr_allocated = 0;
+        if (s->type.t == MACRO_FUNC) {
+            /* NOTE: we do not use next_nomacro to avoid eating the
+               next token. XXX: find better solution */
+        redo:
+            if (macro_ptr) {
+                p = macro_ptr;
+                while (is_space(t = *p) || TOK_LINEFEED == t) 
+                    ++p;
+                if (t == 0 && can_read_stream) {
+                    /* end of macro stream: we must look at the token
+                       after in the file */
+                    struct macro_level *ml = *can_read_stream;
+                    macro_ptr = NULL;
+                    if (ml)
+                    {
+                        macro_ptr = ml->p;
+                        ml->p = NULL;
+                        *can_read_stream = ml -> prev;
+                    }
+                    /* also, end of scope for nested defined symbol */
+                    (*nested_list)->v = -1;
+                    goto redo;
+                }
+            } else {
+                ch = file->buf_ptr[0];
+                while (is_space(ch) || ch == '\n' || ch == '/')
+		  {
+		    if (ch == '/')
+		      {
+			int c;
+			uint8_t *p = file->buf_ptr;
+			PEEKC(c, p);
+			if (c == '*') {
+			    p = parse_comment(p);
+			    file->buf_ptr = p - 1;
+			} else if (c == '/') {
+			    p = parse_line_comment(p);
+			    file->buf_ptr = p - 1;
+			} else
+			  break;
+		      }
+		    cinp();
+		  }
+                t = ch;
+            }
+            if (t != '(') /* no macro subst */
+                return -1;
+                    
+            /* argument macro */
+            next_nomacro();
+            next_nomacro();
+            args = NULL;
+            sa = s->next;
+            /* NOTE: empty args are allowed, except if no args */
+            for(;;) {
+                /* handle '()' case */
+                if (!args && !sa && tok == ')')
+                    break;
+                if (!sa)
+                    tcc_error("macro '%s' used with too many args",
+                          get_tok_str(s->v, 0));
+                tok_str_new(&str);
+                parlevel = spc = 0;
+                /* NOTE: non zero sa->t indicates VA_ARGS */
+                while ((parlevel > 0 || 
+                        (tok != ')' && 
+                         (tok != ',' || sa->type.t))) && 
+                       tok != -1) {
+                    if (tok == '(')
+                        parlevel++;
+                    else if (tok == ')')
+                        parlevel--;
+                    if (tok == TOK_LINEFEED)
+                        tok = ' ';
+                    if (!check_space(tok, &spc))
+                        tok_str_add2(&str, tok, &tokc);
+                    next_nomacro_spc();
+                }
+                str.len -= spc;
+                tok_str_add(&str, 0);
+                sa1 = sym_push2(&args, sa->v & ~SYM_FIELD, sa->type.t, 0);
+                sa1->d = str.str;
+                sa = sa->next;
+                if (tok == ')') {
+                    /* special case for gcc var args: add an empty
+                       var arg argument if it is omitted */
+                    if (sa && sa->type.t && gnu_ext)
+                        continue;
+                    else
+                        break;
+                }
+                if (tok != ',')
+                    expect(",");
+                next_nomacro();
+            }
+            if (sa) {
+                tcc_error("macro '%s' used with too few args",
+                      get_tok_str(s->v, 0));
+            }
+
+            /* now subst each arg */
+            mstr = macro_arg_subst(nested_list, mstr, args);
+            /* free memory */
+            sa = args;
+            while (sa) {
+                sa1 = sa->prev;
+                tok_str_free(sa->d);
+                sym_free(sa);
+                sa = sa1;
+            }
+            mstr_allocated = 1;
+        }
+        sym_push2(nested_list, s->v, 0, 0);
+        macro_subst(tok_str, nested_list, mstr, can_read_stream);
+        /* pop nested defined symbol */
+        sa1 = *nested_list;
+        *nested_list = sa1->prev;
+        sym_free(sa1);
+        if (mstr_allocated)
+            tok_str_free(mstr);
+    }
+    return 0;
+}
+
+/* handle the '##' operator. Return NULL if no '##' seen. Otherwise
+   return the resulting string (which must be freed). */
+static inline int *macro_twosharps(const int *macro_str)
+{
+    const int *ptr;
+    int t;
+    TokenString macro_str1;
+    CString cstr;
+    int n, start_of_nosubsts;
+
+    /* we search the first '##' */
+    for(ptr = macro_str;;) {
+        CValue cval;
+        TOK_GET(&t, &ptr, &cval);
+        if (t == TOK_TWOSHARPS)
+            break;
+        /* nothing more to do if end of string */
+        if (t == 0)
+            return NULL;
+    }
+
+    /* we saw '##', so we need more processing to handle it */
+    start_of_nosubsts = -1;
+    tok_str_new(&macro_str1);
+    for(ptr = macro_str;;) {
+        TOK_GET(&tok, &ptr, &tokc);
+        if (tok == 0)
+            break;
+        if (tok == TOK_TWOSHARPS)
+            continue;
+        if (tok == TOK_NOSUBST && start_of_nosubsts < 0)
+            start_of_nosubsts = macro_str1.len;
+        while (*ptr == TOK_TWOSHARPS) {
+            /* given 'a##b', remove nosubsts preceding 'a' */
+            if (start_of_nosubsts >= 0)
+                macro_str1.len = start_of_nosubsts;
+            /* given 'a##b', skip '##' */
+            t = *++ptr;
+            /* given 'a##b', remove nosubsts preceding 'b' */
+            while (t == TOK_NOSUBST)
+                t = *++ptr;
+            if (t && t != TOK_TWOSHARPS) {
+                CValue cval;
+                TOK_GET(&t, &ptr, &cval);
+                /* We concatenate the two tokens */
+                cstr_new(&cstr);
+                cstr_cat(&cstr, get_tok_str(tok, &tokc));
+                n = cstr.size;
+                cstr_cat(&cstr, get_tok_str(t, &cval));
+                cstr_ccat(&cstr, '\0');
+
+                tcc_open_bf(tcc_state, ":paste:", cstr.size);
+                memcpy(file->buffer, cstr.data, cstr.size);
+                for (;;) {
+                    next_nomacro1();
+                    if (0 == *file->buf_ptr)
+                        break;
+                    tok_str_add2(&macro_str1, tok, &tokc);
+                    tcc_warning("pasting \"%.*s\" and \"%s\" does not give a valid preprocessing token",
+                        n, cstr.data, (char*)cstr.data + n);
+                }
+                tcc_close();
+                cstr_free(&cstr);
+            }
+        }
+        if (tok != TOK_NOSUBST) 
+            start_of_nosubsts = -1;
+        tok_str_add2(&macro_str1, tok, &tokc);
+    }
+    tok_str_add(&macro_str1, 0);
+    return macro_str1.str;
+}
+
+
+/* do macro substitution of macro_str and add result to
+   (tok_str,tok_len). 'nested_list' is the list of all macros we got
+   inside to avoid recursing. */
+static void macro_subst(TokenString *tok_str, Sym **nested_list, 
+                        const int *macro_str, struct macro_level ** can_read_stream)
+{
+    Sym *s;
+    int *macro_str1;
+    const int *ptr;
+    int t, ret, spc;
+    CValue cval;
+    struct macro_level ml;
+    int force_blank;
+    
+    /* first scan for '##' operator handling */
+    ptr = macro_str;
+    macro_str1 = macro_twosharps(ptr);
+
+    if (macro_str1) 
+        ptr = macro_str1;
+    spc = 0;
+    force_blank = 0;
+
+    while (1) {
+        /* NOTE: ptr == NULL can only happen if tokens are read from
+           file stream due to a macro function call */
+        if (ptr == NULL)
+            break;
+        TOK_GET(&t, &ptr, &cval);
+        if (t == 0)
+            break;
+        if (t == TOK_NOSUBST) {
+            /* following token has already been subst'd. just copy it on */
+            tok_str_add2(tok_str, TOK_NOSUBST, NULL);
+            TOK_GET(&t, &ptr, &cval);
+            goto no_subst;
+        }
+        s = define_find(t);
+        if (s != NULL) {
+            /* if nested substitution, do nothing */
+            if (sym_find2(*nested_list, t)) {
+                /* and mark it as TOK_NOSUBST, so it doesn't get subst'd again */
+                tok_str_add2(tok_str, TOK_NOSUBST, NULL);
+                goto no_subst;
+            }
+            ml.p = macro_ptr;
+            if (can_read_stream)
+                ml.prev = *can_read_stream, *can_read_stream = &ml;
+            macro_ptr = (int *)ptr;
+            tok = t;
+            ret = macro_subst_tok(tok_str, nested_list, s, can_read_stream);
+            ptr = (int *)macro_ptr;
+            macro_ptr = ml.p;
+            if (can_read_stream && *can_read_stream == &ml)
+                *can_read_stream = ml.prev;
+            if (ret != 0)
+                goto no_subst;
+            if (parse_flags & PARSE_FLAG_SPACES)
+                force_blank = 1;
+        } else {
+        no_subst:
+            if (force_blank) {
+                tok_str_add(tok_str, ' ');
+                spc = 1;
+                force_blank = 0;
+            }
+            if (!check_space(t, &spc)) 
+                tok_str_add2(tok_str, t, &cval);
+        }
+    }
+    if (macro_str1)
+        tok_str_free(macro_str1);
+}
+
+/* return next token with macro substitution */
+ST_FUNC void next(void)
+{
+    Sym *nested_list, *s;
+    TokenString str;
+    struct macro_level *ml;
+
+ redo:
+    if (parse_flags & PARSE_FLAG_SPACES)
+        next_nomacro_spc();
+    else
+        next_nomacro();
+    if (!macro_ptr) {
+        /* if not reading from macro substituted string, then try
+           to substitute macros */
+        if (tok >= TOK_IDENT &&
+            (parse_flags & PARSE_FLAG_PREPROCESS)) {
+            s = define_find(tok);
+            if (s) {
+                /* we have a macro: we try to substitute */
+                tok_str_new(&str);
+                nested_list = NULL;
+                ml = NULL;
+                if (macro_subst_tok(&str, &nested_list, s, &ml) == 0) {
+                    /* substitution done, NOTE: maybe empty */
+                    tok_str_add(&str, 0);
+                    macro_ptr = str.str;
+                    macro_ptr_allocated = str.str;
+                    goto redo;
+                }
+            }
+        }
+    } else {
+        if (tok == 0) {
+            /* end of macro or end of unget buffer */
+            if (unget_buffer_enabled) {
+                macro_ptr = unget_saved_macro_ptr;
+                unget_buffer_enabled = 0;
+            } else {
+                /* end of macro string: free it */
+                tok_str_free(macro_ptr_allocated);
+                macro_ptr_allocated = NULL;
+                macro_ptr = NULL;
+            }
+            goto redo;
+        } else if (tok == TOK_NOSUBST) {
+            /* discard preprocessor's nosubst markers */
+            goto redo;
+        }
+    }
+    
+    /* convert preprocessor tokens into C tokens */
+    if (tok == TOK_PPNUM &&
+        (parse_flags & PARSE_FLAG_TOK_NUM)) {
+        parse_number((char *)tokc.cstr->data);
+    }
+}
+
+/* push back current token and set current token to 'last_tok'. Only
+   identifier case handled for labels. */
+ST_INLN void unget_tok(int last_tok)
+{
+    int i, n;
+    int *q;
+    if (unget_buffer_enabled)
+      {
+        /* assert(macro_ptr == unget_saved_buffer + 1);
+	   assert(*macro_ptr == 0);  */
+      }
+    else
+      {
+	unget_saved_macro_ptr = macro_ptr;
+	unget_buffer_enabled = 1;
+      }
+    q = unget_saved_buffer;
+    macro_ptr = q;
+    *q++ = tok;
+    n = tok_ext_size(tok) - 1;
+    for(i=0;i<n;i++)
+        *q++ = tokc.tab[i];
+    *q = 0; /* end of token string */
+    tok = last_tok;
+}
+
+
+/* better than nothing, but needs extension to handle '-E' option
+   correctly too */
+ST_FUNC void preprocess_init(TCCState *s1)
+{
+    s1->include_stack_ptr = s1->include_stack;
+    /* XXX: move that before to avoid having to initialize
+       file->ifdef_stack_ptr ? */
+    s1->ifdef_stack_ptr = s1->ifdef_stack;
+    file->ifdef_stack_ptr = s1->ifdef_stack_ptr;
+
+    vtop = vstack - 1;
+    s1->pack_stack[0] = 0;
+    s1->pack_stack_ptr = s1->pack_stack;
+}
+
+ST_FUNC void preprocess_new(void)
+{
+    int i, c;
+    const char *p, *r;
+
+    /* init isid table */
+    for(i=CH_EOF;i<256;i++)
+        isidnum_table[i-CH_EOF] = isid(i) || isnum(i);
+
+    /* add all tokens */
+    table_ident = NULL;
+    memset(hash_ident, 0, TOK_HASH_SIZE * sizeof(TokenSym *));
+    
+    tok_ident = TOK_IDENT;
+    p = tcc_keywords;
+    while (*p) {
+        r = p;
+        for(;;) {
+            c = *r++;
+            if (c == '\0')
+                break;
+        }
+        tok_alloc(p, r - p - 1);
+        p = r;
+    }
+}
+
+/* Preprocess the current file */
+ST_FUNC int tcc_preprocess(TCCState *s1)
+{
+    Sym *define_start;
+
+    BufferedFile *file_ref, **iptr, **iptr_new;
+    int token_seen, line_ref, d;
+    const char *s;
+
+    preprocess_init(s1);
+    define_start = define_stack;
+    ch = file->buf_ptr[0];
+    tok_flags = TOK_FLAG_BOL | TOK_FLAG_BOF;
+    parse_flags = PARSE_FLAG_ASM_COMMENTS | PARSE_FLAG_PREPROCESS |
+        PARSE_FLAG_LINEFEED | PARSE_FLAG_SPACES;
+    token_seen = 0;
+    line_ref = 0;
+    file_ref = NULL;
+    iptr = s1->include_stack_ptr;
+
+    for (;;) {
+        next();
+        if (tok == TOK_EOF) {
+            break;
+        } else if (file != file_ref) {
+            goto print_line;
+        } else if (tok == TOK_LINEFEED) {
+            if (!token_seen)
+                continue;
+            ++line_ref;
+            token_seen = 0;
+        } else if (!token_seen) {
+            d = file->line_num - line_ref;
+            if (file != file_ref || d < 0 || d >= 8) {
+print_line:
+                iptr_new = s1->include_stack_ptr;
+                s = iptr_new > iptr ? " 1"
+                  : iptr_new < iptr ? " 2"
+                  : iptr_new > s1->include_stack ? " 3"
+                  : ""
+                  ;
+                iptr = iptr_new;
+                fprintf(s1->ppfp, "# %d \"%s\"%s\n", file->line_num, file->filename, s);
+            } else {
+                while (d)
+                    fputs("\n", s1->ppfp), --d;
+            }
+            line_ref = (file_ref = file)->line_num;
+            token_seen = tok != TOK_LINEFEED;
+            if (!token_seen)
+                continue;
+        }
+        fputs(get_tok_str(tok, &tokc), s1->ppfp);
+    }
+    free_defines(define_start);
+    return 0;
+}