qtbindings 4.6.3.0-x86-mingw32

data/ext/generator/parser/rpp/pp-scanner.cpp

@@ -0,0 +1,310 @@
+/*
+  Copyright 2005 Roberto Raggi <roberto@kdevelop.org>
+  Copyright 2006 Hamish Rodda <rodda@kde.org>
+
+  Permission to use, copy, modify, distribute, and sell this software and its
+  documentation for any purpose is hereby granted without fee, provided that
+  the above copyright notice appear in all copies and that both that
+  copyright notice and this permission notice appear in supporting
+  documentation.
+
+  The above copyright notice and this permission notice shall be included in
+  all copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+  KDEVELOP TEAM BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+  AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+  CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/
+
+#include "pp-scanner.h"
+#include "chartools.h"
+#include "../indexedstring.h"
+#include "../kdevvarlengtharray.h"
+
+using namespace rpp;
+
+void pp_skip_blanks::operator()(Stream& input, Stream& output)
+{
+  while (!input.atEnd()) {
+    if (input == '\\') {
+      ++input;
+      if (input != '\n') {
+        --input;
+        return;
+
+      } else {
+        ++input;
+        continue;
+      }
+    }
+
+    if (input == '\n' || !isSpace(input.current()))
+      return;
+
+    output << input;
+    ++input;
+  }
+}
+
+void pp_skip_whitespaces::operator()(Stream& input, Stream& output)
+{
+  while (!input.atEnd()) {
+    if (!isSpace(input.current()))
+      return;
+
+    output << input;
+    ++input;
+  }
+}
+
+void pp_skip_comment_or_divop::operator()(Stream& input, Stream& output, bool outputText)
+{
+  enum {
+    MAYBE_BEGIN,
+    BEGIN,
+    MAYBE_END,
+    END,
+    IN_COMMENT,
+    IN_CXX_COMMENT
+  } state (MAYBE_BEGIN);
+
+  while (!input.atEnd()) {
+    switch (state) {
+      case MAYBE_BEGIN:
+        if (input != '/')
+          return;
+
+        state = BEGIN;
+        break;
+
+      case BEGIN:
+        if (input == '*')
+          state = IN_COMMENT;
+        else if (input == '/')
+          state = IN_CXX_COMMENT;
+        else
+          return;
+        break;
+
+      case IN_COMMENT:
+        if (input == '*')
+          state = MAYBE_END;
+        break;
+
+      case IN_CXX_COMMENT:
+        if (input == '\n')
+          return;
+        break;
+
+      case MAYBE_END:
+        if (input == '/')
+          state = END;
+        else if (input != '*')
+          state = IN_COMMENT;
+        break;
+
+      case END:
+        return;
+    }
+
+    if (outputText) {
+      output << input;
+      ++input;
+
+    } else if (input == '\n') {
+      output << '\n';
+      ++input;
+      output.mark(input.inputPosition());
+
+    } else {
+      output << ' ';
+      ++input;
+    }
+  }
+}
+
+uint pp_skip_identifier::operator()(Stream& input)
+{
+  KDevVarLengthArray<char, 100> identifier;
+  
+  IndexedString::RunningHash hash;
+
+  while (!input.atEnd()) {
+    if(!isCharacter(input.current())) {
+      //Do a more complex merge, where also tokenized identifiers can be merged
+      IndexedString ret;
+      if(!identifier.isEmpty())
+        ret = IndexedString(identifier.constData(), identifier.size(), hash.hash);
+      
+      while (!input.atEnd()) {
+        uint current = input.current();
+        
+        if (!isLetterOrNumber(current) && input != '_' && isCharacter(current))
+          break;
+        
+        if(ret.isEmpty())
+          ret = IndexedString::fromIndex(current); //The most common fast path
+        else ///@todo Be better to build up a complete buffer and then append it all, so we don't get he intermediate strings into the repository
+          ret = IndexedString(ret.byteArray() + IndexedString::fromIndex(input.current()).byteArray());
+        
+        ++input;
+      }
+      return ret.index();
+    }
+    //Collect characters and connect them to an IndexedString
+    
+    if (!isLetterOrNumber(input.current()) && input != '_')
+        break;
+
+    char c = characterFromIndex(input);
+    hash.append(c);
+    identifier.append(c);
+    ++input;
+  }
+
+  return IndexedString(identifier.constData(), identifier.size(), hash.hash).index();
+}
+
+void pp_skip_number::operator()(Stream& input, Stream& output)
+{
+  while (!input.atEnd()) {
+    if (!isLetterOrNumber(input.current()) && input != '_')
+        return;
+
+    output << input;
+    ++input;
+  }
+}
+
+void pp_skip_string_literal::operator()(Stream& input, Stream& output)
+{
+  enum {
+    BEGIN,
+    IN_STRING,
+    QUOTE,
+    END
+  } state (BEGIN);
+
+  while (!input.atEnd()) {
+    switch (state) {
+      case BEGIN:
+        if (input != '\"')
+          return;
+        state = IN_STRING;
+        break;
+
+      case IN_STRING:
+//         Q_ASSERT(input != '\n');
+
+        if (input == '\"')
+          state = END;
+        else if (input == '\\')
+          state = QUOTE;
+        break;
+
+      case QUOTE:
+        state = IN_STRING;
+        break;
+
+      case END:
+        return;
+    }
+
+    output << input;
+    ++input;
+  }
+}
+
+void pp_skip_char_literal::operator()(Stream& input, Stream& output)
+{
+  enum {
+    BEGIN,
+    IN_STRING,
+    QUOTE,
+    END
+  } state (BEGIN);
+  int inner_count = 0;
+
+  while (!input.atEnd()) {
+    if (state == END)
+      break;
+
+    switch (state) {
+      case BEGIN:
+        if (input != '\'')
+          return;
+        state = IN_STRING;
+        break;
+
+      case IN_STRING:
+        if(input == '\n' || inner_count > 3)
+          return; //Probably this isn't a string literal. Example: "#warning What's up"
+
+        if (input == '\'')
+          state = END;
+        else if (input == '\\')
+          state = QUOTE;
+        break;
+        ++inner_count;
+      case QUOTE:
+        state = IN_STRING;
+        break;
+
+      default:
+        Q_ASSERT(0);
+        break;
+    }
+
+    output << input;
+    ++input;
+  }
+}
+
+///@todo Can this deal with comments? like /*(*/
+void pp_skip_argument::operator()(Stream& input, Stream& output)
+{
+  int depth = 0;
+
+  while (!input.atEnd()) {
+    if (!depth && (input == ')' || input == ',')) {
+      return;
+
+    } else if (input == '(') {
+      ++depth;
+
+    } else if (input == ')') {
+      --depth;
+
+    } else if (input == '\"') {
+      skip_string_literal(input, output);
+      continue;
+
+    } else if (input == '\'') {
+      skip_char_literal (input, output);
+      continue;
+
+    } else if (input == '/') {
+      skip_comment_or_divop (input, output, true);
+      continue;
+
+    } else if (isLetter(input.current()) || input == '_') {
+      Anchor inputPosition = input.inputPosition();
+      output.appendString(inputPosition, IndexedString::fromIndex(skip_identifier(input)));
+      continue;
+
+    } else if (isNumber(input.current())) {
+      output.mark(input.inputPosition());
+      skip_number(input, output);
+      continue;
+
+    }
+
+    output << input;
+    ++input;
+  }
+
+  return;
+}

data/ext/generator/parser/rpp/pp-scanner.h

@@ -0,0 +1,97 @@
+/*
+  Copyright 2005 Roberto Raggi <roberto@kdevelop.org>
+  Copyright 2006 Hamish Rodda <rodda@kde.org>
+
+  Permission to use, copy, modify, distribute, and sell this software and its
+  documentation for any purpose is hereby granted without fee, provided that
+  the above copyright notice appear in all copies and that both that
+  copyright notice and this permission notice appear in supporting
+  documentation.
+
+  The above copyright notice and this permission notice shall be included in
+  all copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+  KDEVELOP TEAM BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+  AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+  CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/
+
+#ifndef PP_SCANNER_H
+#define PP_SCANNER_H
+
+
+
+#include "pp-stream.h"
+
+namespace rpp {
+
+class pp_skip_blanks
+{
+public:
+  void operator()(Stream& input, Stream& output);
+};
+
+class pp_skip_whitespaces
+{
+public:
+  void operator()(Stream& input, Stream& output);
+};
+
+class pp_skip_comment_or_divop
+{
+public:
+  /**
+   * This scanner can either output equivalent blank space,
+   * or the actual text (the default).
+   */
+  void operator()(Stream& input, Stream& output, bool outputText = false);
+
+private:
+  bool m_outputText;
+};
+
+class pp_skip_identifier
+{
+public:
+  uint operator()(Stream& input);
+};
+
+class pp_skip_number
+{
+public:
+  void operator()(Stream& input, Stream& output);
+};
+
+class pp_skip_string_literal
+{
+public:
+  void operator()(Stream& input, Stream& output);
+};
+
+class pp_skip_char_literal
+{
+public:
+  void operator()(Stream& input, Stream& output);
+};
+
+//Does also skip any whitespace behind the argument
+class pp_skip_argument
+{
+public:
+  void operator()(Stream& input, Stream& output);
+
+private:
+  pp_skip_number skip_number;
+  pp_skip_identifier skip_identifier;
+  pp_skip_string_literal skip_string_literal;
+  pp_skip_char_literal skip_char_literal;
+  pp_skip_comment_or_divop skip_comment_or_divop;
+};
+
+}
+
+#endif // PP_SCANNER_H
+

data/ext/generator/parser/rpp/pp-stream.cpp

@@ -0,0 +1,425 @@
+/*
+  Copyright 2006 Hamish Rodda <rodda@kde.org>
+  Copyright 2008-2009 David Nolden <david.nolden.kdevelop@art-master.de>
+
+  Permission to use, copy, modify, distribute, and sell this software and its
+  documentation for any purpose is hereby granted without fee, provided that
+  the above copyright notice appear in all copies and that both that
+  copyright notice and this permission notice appear in supporting
+  documentation.
+
+  The above copyright notice and this permission notice shall be included in
+  all copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+  KDEVELOP TEAM BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+  AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+  CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/
+
+#include "pp-stream.h"
+
+///@todo Better splitting of input and output functionality
+
+#include "pp-location.h"
+#include "chartools.h"
+#include "../indexedstring.h"
+
+using namespace rpp;
+
+const unsigned int Stream::newline(indexFromCharacter('\n'));
+static unsigned int nullItem(0);
+const unsigned int deepLine(indexFromCharacter('_'));
+
+// bool shouldMergeTo(uint previous) {
+//   return !isCharacter(previous) || previous == deepLine || isLetter(previous);
+// }
+// 
+// bool shouldMerge(uint follower) {
+//   return !isCharacter(follower) || follower == deepLine || isLetterOrNumber(follower);
+// }
+
+Stream::Stream()
+  : m_string(new PreprocessedContents())
+  , m_isNull(true)
+  , m_skippedToEnd(false)
+  , m_inputPositionLocked(false)
+  , m_onwsString(true)
+  , m_macroExpansion(SimpleCursor::invalid())
+  , m_pos(0)
+  , m_inputLine(0)
+  , m_inputLineStartedAt(0)
+  , m_locationTable(0L)
+  , m_originalInputPosition(SimpleCursor::invalid())
+{
+  end = 0;
+}
+
+Stream::Stream( PreprocessedContents * string, const Anchor& offset, LocationTable* table )
+  : m_string(string)
+  , m_isNull(false)
+  , m_skippedToEnd(false)
+  , m_inputPositionLocked(false)
+  , m_onwsString(false)
+  , m_macroExpansion(SimpleCursor::invalid())
+  , m_pos(0)
+  , m_inputLine(offset.line)
+  , m_inputLineStartedAt(-offset.column)
+  , m_locationTable(table)
+  , m_originalInputPosition(SimpleCursor::invalid())
+{
+  if(offset.collapsed)
+    m_inputPositionLocked = true;
+  c = m_string->constData();
+  end = m_string->constData() + m_string->size();
+}
+
+Stream::Stream( const uint * string, uint stringSize, const Anchor& offset, LocationTable* table )
+  : m_string(new PreprocessedContents(stringSize))
+  , m_isNull(false)
+  , m_skippedToEnd(false)
+  , m_inputPositionLocked(false)
+  , m_onwsString(true)
+  , m_macroExpansion(SimpleCursor::invalid())
+  , m_pos(0)
+  , m_inputLine(offset.line)
+  , m_inputLineStartedAt(-offset.column)
+  , m_locationTable(table)
+  , m_originalInputPosition(SimpleCursor::invalid())
+{
+  memcpy(m_string->data(), string, stringSize * sizeof(uint));
+  if(offset.collapsed)
+    m_inputPositionLocked = true;
+  c = m_string->constData();
+  end = m_string->constData() + m_string->size();
+}
+
+Stream::Stream( PreprocessedContents * string, LocationTable* table )
+  : m_string(string)
+  , m_isNull(false)
+  , m_skippedToEnd(false)
+  , m_inputPositionLocked(false)
+  , m_onwsString(false)
+  , m_macroExpansion(SimpleCursor::invalid())
+  , m_pos(0)
+  , m_inputLine(0)
+  , m_inputLineStartedAt(0)
+  , m_locationTable(table)
+  , m_originalInputPosition(SimpleCursor::invalid())
+{
+  c = m_string->constData();
+  end = m_string->constData() + m_string->size();
+}
+
+Stream::~Stream()
+{
+  if (m_onwsString)
+    delete m_string;
+}
+
+Stream& Stream::operator--()
+{
+  if (c == m_string->constData())
+    return *this;
+
+  --c;
+  --m_pos;
+
+  if(m_inputPositionLocked)
+    --m_inputLineStartedAt;
+  else
+    m_inputLineStartedAt -= (1-IndexedString::fromIndex(*c).length());
+
+
+  return *this;
+}
+
+bool Stream::atEnd() const
+{
+  return c == end;
+}
+
+void Stream::toEnd()
+{
+  m_skippedToEnd = true;
+  c = end;
+}
+
+bool Stream::skippedToEnd() const
+{
+  return m_skippedToEnd;
+}
+
+const uint& Stream::peek(uint offset) const
+{
+  if (c + offset >= end)
+    return nullItem;
+
+  return *(c + offset);
+}
+
+int Stream::offset( ) const
+{
+  return m_pos;
+}
+
+void Stream::seek(int offset)
+{
+  if(m_inputPositionLocked){
+    m_inputLineStartedAt = offset + (m_inputLineStartedAt - m_pos);
+  }else{
+    if(offset < m_pos) {
+      for(int a = offset; a < m_pos; ++a)
+        m_inputLineStartedAt -= (1-IndexedString::fromIndex(m_string->at(a)).length());
+    }else{
+      for(int a = m_pos; a < offset; ++a)
+        m_inputLineStartedAt += (1-IndexedString::fromIndex(m_string->at(a)).length());
+    }
+  }
+  
+  c = m_string->constData() + offset;
+  m_pos = offset;
+  if (c > end) {
+    c = end;
+    m_pos = m_string->size();
+  }
+}
+
+Stream & Stream::operator<< ( const unsigned int& c )
+{
+  // Keep in sync with below
+  if (!m_isNull) {
+    
+/*    if(m_pos > 0) {
+      unsigned int& previous( (*m_string)[m_pos-1] );
+      if(shouldMergeTo(previous)) {
+        if(shouldMerge(c)) {
+          //We have to merge with the previous character, so we get a correct tokenization. This should not happen too often.
+          previous = IndexedString(IndexedString(previous).byteArray() + IndexedString(c).byteArray()).index();
+          kDebug() << "merging" << IndexedString(previous).str() << "and" << IndexedString(c).str();
+          return *this;
+        }
+      }
+    }*/
+    
+    ++m_pos;
+
+    if (c == newline) {
+      ++m_inputLine;
+      m_inputLineStartedAt = m_pos; ///@todo remove
+    }
+
+    m_string->append(c);
+  }
+  return *this;
+}
+
+unsigned int rpp::Stream::popLastOutput() {
+  unsigned int ret = m_string->last();
+  m_string->pop_back();
+  --m_pos;
+  return ret;
+}
+
+unsigned int rpp::Stream::peekLastOutput(uint backOffset) const {
+  if(m_pos - backOffset > 0)
+    return m_string->at(m_pos - backOffset - 1);
+  return 0;
+}
+
+Stream& Stream::operator<< ( const Stream& input )
+{
+  const uint c = input;
+
+  // Keep in sync with above
+  if (!m_isNull) {
+    
+/*    if(m_pos > 0) {
+      unsigned int& previous( (*m_string)[m_pos-1] );
+      if(shouldMergeTo(previous)) {
+        if(shouldMerge(c)) {
+          //We have to merge with the previous character, so we get a correct tokenization. This should not happen too often.
+          previous = IndexedString(IndexedString(previous).byteArray() + IndexedString(c).byteArray()).index();
+          kDebug() << "merging" << IndexedString(previous).str() << "and" << IndexedString(c).str();
+          return *this;
+        }
+      }
+    }*/
+    
+    ++m_pos;
+
+    m_string->append(c);
+    
+    if (c == newline) {
+      Anchor inputPosition = input.inputPosition();
+      ++m_inputLine;
+      m_inputLineStartedAt = m_pos; ///@todo remove
+      if(!inputPosition.collapsed)
+        mark(Anchor(inputPosition.line + 1, 0, false, m_macroExpansion));
+    }
+  }
+  return *this;
+}
+
+Stream& Stream::appendString( const Anchor& inputPosition, const PreprocessedContents & string )
+{
+  if (!isNull()) {
+    
+//     uint offset = 0;
+    
+/*    if(m_pos > 0 && string.size()) {
+      //Eventually merge the tokens. This shouldn't happen too often
+      int size = string.size();
+      unsigned int& previous( (*m_string)[m_pos-1] );
+      for(int a = 0; a < size; ++a) {
+        if(shouldMergeTo(previous)) {
+          if(shouldMerge(string[a])) {
+            //We have to merge with the previous character, so we get a correct tokenization. This should not happen too often.
+            previous = IndexedString(IndexedString(previous).byteArray() + IndexedString(string[a]).byteArray()).index();
+            ++offset;
+            kDebug() << "merging" << IndexedString(previous).str() << "and" << IndexedString(string[a]).str();
+            continue;
+          }
+        }
+        break;
+      }
+    }*/
+    
+   // if(!offset) ///@todo think about his. We lose the input position, but on the other hand the merging should only happen when ## was used
+      mark(inputPosition);
+    
+    *m_string+= string;
+
+    int extraLines = 0;
+    for (int i = 0; i < string.size(); ++i) {
+      
+      if (string.at(i) == newline) {
+        m_pos += i + 1; //Move the current offset to that position, so the marker is set correctly
+        if(!inputPosition.collapsed)
+          mark(Anchor(inputPosition.line + ++extraLines, 0, false, m_macroExpansion));
+        m_pos -= i + 1;
+      }
+    }
+
+    m_pos += string.size();
+
+    // TODO check correctness Probably remove
+    m_inputLineStartedAt = m_pos - (string.size() - string.lastIndexOf(newline)); ///@todo remove
+  }
+  return *this;
+}
+
+Stream& Stream::appendString( const Anchor& inputPosition, IndexedString string )
+{
+
+  if (!isNull()) {
+    
+/*    if(m_pos > 0) {
+      //Eventually merge
+      unsigned int& previous( (*m_string)[m_pos-1] );
+      if(shouldMergeTo(previous)) {
+        if(shouldMerge(string.index())) {
+          //We have to merge with the previous character, so we get a correct tokenization. This should not happen too often.
+          previous = IndexedString(IndexedString(previous).byteArray() + string.byteArray()).index();
+          kDebug() << "merging" << IndexedString(previous).str() << "and" << string.str();
+          return *this; ///We lose the input-position, but on the other hand we would lose it anyway on another level
+        }
+      }
+    }*/
+    
+    mark(inputPosition);
+    m_string->append(string.index());
+
+    int extraLines = 0;
+    if (string.index() == newline) {
+      m_pos += 1; //Move the current offset to that position, so the marker is set correctly
+      if(!inputPosition.collapsed)
+        mark(Anchor(inputPosition.line + ++extraLines, 0, false, m_macroExpansion));
+      m_pos -= 1;
+    }
+
+    m_pos += 1;
+
+    // TODO check correctness Probably remove
+    m_inputLineStartedAt = m_pos; ///@todo remove
+  }
+  return *this;
+}
+
+bool Stream::isNull() const
+{
+  return m_isNull;
+}
+
+Anchor Stream::inputPosition() const
+{
+  return Anchor(m_inputLine, m_pos - m_inputLineStartedAt, m_inputPositionLocked, m_macroExpansion);
+}
+
+void Stream::setInputPosition(const Anchor& position)
+{
+  m_inputLine = position.line;
+  m_inputLineStartedAt = m_pos - position.column;
+  m_inputPositionLocked = position.collapsed;
+}
+
+void Stream::setMacroExpansion(const SimpleCursor& expansion)
+{
+  m_macroExpansion = expansion;
+}
+
+SimpleCursor Stream::macroExpansion() const
+{
+  return m_macroExpansion;
+}
+
+rpp::Anchor rpp::Stream::currentOutputAnchor() const {
+  if(m_locationTable)
+    return m_locationTable->positionAt(m_pos, *m_string).first;
+  return rpp::Anchor();
+}
+
+void Stream::mark(const Anchor& position)
+{
+  Q_ASSERT(m_pos <= m_string->size());
+  if (m_locationTable) {
+    if(m_macroExpansion.isValid()) {
+      Anchor a(position);
+      a.macroExpansion = m_macroExpansion;
+      m_locationTable->anchor(m_pos, a, m_string);
+    }else{
+      m_locationTable->anchor(m_pos, position, m_string);
+    }
+  }
+}
+
+void Stream::reset( )
+{
+  c = m_string->constData();
+  m_inputLineStartedAt = m_inputLine = m_pos = 0;
+  m_inputPositionLocked = false;
+}
+
+QByteArray rpp::Stream::stringFrom(int offset) const
+{
+  QByteArray ret;
+  for(int a = offset; a < m_pos; ++a)
+    ret += IndexedString::fromIndex((*m_string)[a]).byteArray();
+  
+  return ret;
+}
+
+SimpleCursor rpp::Stream::originalInputPosition() const
+{
+  if (m_originalInputPosition.isValid())
+    return m_originalInputPosition;
+
+  return inputPosition();
+}
+
+void rpp::Stream::setOriginalInputPosition(const SimpleCursor & position)
+{
+  m_originalInputPosition = position;
+}