hdt 3.2.0 → 3.2.1

package/deps/hdt-it/hdtcachedinfo.cpp

@@ -0,0 +1,155 @@
+#include "hdtcachedinfo.hpp"
+#include "constants.h"
+
+#include "../../hdt-lib/src/triples/BitmapTriples.hpp"
+
+#include <fstream>
+
+HDTCachedInfo::HDTCachedInfo(hdt::HDT *hdt) : hdt(hdt)
+{
+
+}
+
+void HDTCachedInfo::generateGeneralInfo(hdt::ProgressListener *listener)
+{
+    // Count predicates
+    unsigned int nPred = hdt->getDictionary()->getNpredicates();
+
+    maxPredicateCount = 0;
+    predicateCount.clear();
+    predicateCount.resize(nPred+1);
+
+   NOTIFYCOND(listener, "Loading predicates", 0, 100);
+
+	// TODO: Use predicateCount directly
+    if(hdt->isIndexed()) {
+        hdt::Triples *t = hdt->getTriples();
+        hdt::TripleID triplePredicate;
+        for(int p=1;p<=nPred;p++) {
+#if 1
+            predicateCount[p] = t->getNumAppearances(p);
+#else
+            triplePredicate.setAll(0, p, 0);
+            hdt::IteratorTripleID *predIt = t->search(triplePredicate);
+
+            predicateCount[p] = predIt->estimatedNumResults();
+
+            maxPredicateCount = max(maxPredicateCount, predicateCount[p]);
+
+            delete predIt;
+#endif
+        }
+    }
+
+    // Calculate Predicate Colors
+    predicateColors.clear();
+    predicateColors.resize(nPred);
+    ColorRamp2 cr;
+    for(unsigned int i=0;i<nPred; i++) {
+        cr.apply(&predicateColors[i], i, 0, nPred-1);
+    }
+
+    // Go over first 50 results to make sure they are loaded.
+    hdt::TripleString pat;
+    hdt::IteratorTripleString *it2 = hdt->search(pat);
+    for(int i=0;i<50 && it2->hasNext(); i++) {
+        it2->next();
+    }
+    delete it2;
+
+    // Iterate over elements of the array to make sure their dictionary entries are loaded.
+#if 0
+    hdt::TripleString out;
+    for(size_t i=0;i<triples.size();i++) {
+	hdt->getDictionary()->tripleIDtoTripleString(triples[i], out);
+    }
+#endif
+}
+
+void HDTCachedInfo::generateMatrix(hdt::ProgressListener *listener)
+{
+    // Generate matrix
+    hdt::Triples *t = hdt->getTriples();
+    size_t increment = t->getNumberOfElements()/RENDER_NUM_POINTS;
+    increment = increment < 1 ? 1 : increment;
+
+
+// Using InterleavedIterator to jump directly to the triple and get the related information.
+    hdt::BTInterleavedIterator it(dynamic_cast<hdt::BitmapTriples *>(t), increment);
+
+    size_t count=0;
+    while(it.hasNext()) {
+        hdt::TripleID *tid = it.next();
+        triples.push_back(*tid);
+        //cout << *tid << endl;
+
+        NOTIFYCOND(listener, "Generating Matrix", count, RENDER_NUM_POINTS);
+        count++;
+        //if((count%100)==0)
+            //cout << "Iteration: " << count << endl;
+    }
+
+// Using normal iterator that goes through all entries.
+//    hdt::IteratorTripleID *it = t->searchAll();
+//    size_t count=0;
+//    for(size_t i=0;i<t->getNumberOfElements();i+=increment) {
+//        it->goTo(i);
+//        hdt::TripleID *tid = it->next();
+//        triples.push_back(*tid);
+//        NOTIFYCOND(listener, "Generating Matrix", i, t->getNumberOfElements());
+//        count++;
+//        if((count%100)==0)
+//            cout << "Iteration: " << count << endl;
+//    }
+//    delete it;
+
+}
+
+Color * HDTCachedInfo::getPredicateColor(size_t npred)
+{
+    return &predicateColors[npred];
+}
+
+size_t HDTCachedInfo::getPredicateUsages(size_t predicate)
+{
+    return predicateCount[predicate];
+}
+
+size_t HDTCachedInfo::getMaxPredicateCount()
+{
+    return maxPredicateCount;
+}
+
+vector<hdt::TripleID> &HDTCachedInfo::getTriples()
+{
+    return triples;
+}
+
+void HDTCachedInfo::save(QString &fileName, hdt::ProgressListener *listener)
+{
+	// Only save info of files bigger than 2M triples. Otherwise is fast to create from scratch.
+	if(hdt->getTriples()->getNumberOfElements()>2000000) {
+	    std::ofstream out(fileName.toLatin1(), ios::binary);
+        uint64_t numTriples = triples.size();
+        out.write((char *)&numTriples, sizeof(uint64_t));
+	    out.write((char *)&triples[0], sizeof(hdt::TripleID)*numTriples);
+	    out.close();
+	}
+}
+
+void HDTCachedInfo::load(QString &fileName, hdt::ProgressListener *listener)
+{
+    generateGeneralInfo(listener);
+
+    std::ifstream in(fileName.toLatin1(), ios::binary);
+    if(in.good()) {
+        uint64_t numTriples;
+        in.read((char *)&numTriples, sizeof(uint64_t));
+        triples.resize(numTriples);
+        in.read((char *)&triples[0], sizeof(hdt::TripleID)*numTriples);
+        in.close();
+    } else {
+        generateMatrix(listener);
+        save(fileName, listener);
+    }
+}

package/deps/hdt-it/hdtcachedinfo.hpp

@@ -0,0 +1,38 @@
+#ifndef HDTCACHEDINFO_HPP
+#define HDTCACHEDINFO_HPP
+
+#include <QString>
+#include <HDT.hpp>
+#include <HDTListener.hpp>
+#include "StopWatch.hpp"
+
+#include "Color.h"
+
+
+class HDTCachedInfo
+{
+private:
+
+    hdt::HDT *hdt;
+    vector<hdt::TripleID> triples;
+
+    vector<Color> predicateColors;
+    size_t maxPredicateCount;
+    vector<size_t> predicateCount;
+
+public:
+    HDTCachedInfo(hdt::HDT *hdt);
+
+    Color *getPredicateColor(size_t npred);
+    size_t getPredicateUsages(size_t predicate);
+    size_t getMaxPredicateCount();
+    vector<hdt::TripleID> &getTriples();
+
+    void generateGeneralInfo(hdt::ProgressListener *listener=NULL);
+    void generateMatrix(hdt::ProgressListener *listener=NULL);
+
+    void save(QString &fileName, hdt::ProgressListener *listener=NULL);
+    void load(QString &fileName, hdt::ProgressListener *listener=NULL);
+};
+
+#endif // HDTCACHEDINFO_HPP

package/deps/hdt-it/hdtcontroller.cpp

@@ -0,0 +1,369 @@
+#include <iostream>
+#include <fstream>
+
+#include <inttypes.h>
+
+#include <HDTManager.hpp>
+#include <HDTVocabulary.hpp>
+#include <SingleTriple.hpp>
+
+#include <QtGui>
+
+#include "hdtcontroller.hpp"
+#include "hdtoperation.hpp"
+#include "hdtcachedinfo.hpp"
+#include "stringutils.hpp"
+#include "resultcounter.hpp"
+
+#define SHOW_DIALOGS
+
+HDTController::HDTController(QObject *parent) :
+    QObject(parent),
+    hdt(NULL),
+    searchPatternID(0,0,0),
+    selectedTriple(0,0,0),
+    numResults(0),
+    iteratorResults(NULL)
+{
+    subjectModel = new TripleComponentModel(this, hdt::SUBJECT);
+    predicateModel = new TripleComponentModel(this, hdt::PREDICATE);
+    objectModel = new TripleComponentModel(this, hdt::OBJECT);
+    searchResultsModel = new SearchResultsModel(this);
+    headerModel = new HeaderModel(this);
+    predicateStatus = new PredicateStatus(this);
+    regexModel = new RegexModel(this);
+}
+
+HDTController::~HDTController() {
+    closeHDT();
+    delete subjectModel;
+    delete predicateModel;
+    delete objectModel;
+    delete searchResultsModel;
+    delete headerModel;
+    delete predicateStatus;
+    delete regexModel;
+}
+
+void HDTController::updateOnHDTChanged()
+{
+    this->numResults = hdt->getTriples()->getNumberOfElements();
+
+    subjectModel->notifyLayoutChanged();
+    predicateModel->notifyLayoutChanged();
+    predicateStatus->refreshAll();
+    objectModel->notifyLayoutChanged();
+    searchResultsModel->updateResultListChanged();
+    headerModel->updateDatasetChanged();
+    regexModel->updateDatasetChanged();
+
+    emit datasetChanged();
+}
+
+void HDTController::importRDFFile(QString file, const string &baseUri, hdt::RDFNotation notation, hdt::HDTSpecification &spec)
+{
+    closeHDT();
+
+    HDTOperation *hdtop = new HDTOperation();
+    hdtop->loadFromRDF(spec, file, notation, baseUri);
+    int result = hdtop->exec();
+    if(result==0) {
+        this->hdt = hdtop->getHDT();
+        this->hdtCachedInfo = hdtop->getHDTInfo();
+        this->fileName = file;
+
+        updateOnHDTChanged();
+    }
+    delete hdtop;
+}
+
+void HDTController::openHDTFile(QString file)
+{
+    closeHDT();
+
+    HDTOperation *hdtop = new HDTOperation(file);
+    hdtop->loadFromHDT(file);
+    int result = hdtop->exec();
+
+    if(result==0) {
+        this->hdt = hdtop->getHDT();
+        this->hdtCachedInfo = hdtop->getHDTInfo();
+        this->fileName = file;
+
+        updateOnHDTChanged();
+    }
+
+    delete hdtop;
+}
+
+void HDTController::saveHDTFile(QString file)
+{
+    if(hdt!=NULL) {
+#ifdef SHOW_DIALOGS
+        HDTOperation *hdtop = new HDTOperation(hdt);
+        hdtop->saveToHDT(file);
+        hdtop->exec();
+        delete hdtop;
+#else
+        hdt->saveToHDT(file.toAscii());
+#endif
+    }
+}
+
+void HDTController::exportRDFFile(QString file, hdt::RDFNotation notation)
+{
+    if(hdt!=NULL) { 
+#ifdef SHOW_DIALOGS
+        HDTOperation *hdtop = new HDTOperation(hdt);
+        hdtop->saveToRDF(file, notation);
+        hdtop->exec();
+        delete hdtop;
+#else
+        hdt::RDFSerializer *serializer = hdt::RDFSerializer::getSerializer(file.toAscii(), notation);
+        hdt->saveToRDF(*serializer);
+#endif
+    }
+}
+
+void HDTController::exportResultsRDFFile(QString file, hdt::RDFNotation notation)
+{
+    if(hdt!=NULL) {
+#ifdef SHOW_DIALOGS
+        HDTOperation *hdtop = new HDTOperation(hdt);
+        hdt::IteratorTripleString *it = hdt->search(searchPatternString);
+        hdtop->exportResults(file, it, getNumResults(), notation);
+        hdtop->exec();
+        delete hdtop;
+#else
+        hdt::RDFSerializer *serializer = hdt::RDFSerializer::getSerializer(file.toAscii(), notation);
+        hdt::IteratorTripleString *it = hdt->search(searchPatternString);
+        serializer->serialize(iterator, getNumResults());
+        delete serializer;
+        delete iterator;
+        break;
+#endif
+    }
+}
+
+
+void HDTController::closeHDT()
+{
+    if(hdt!=NULL) {
+        delete hdt;
+        hdt = NULL;
+
+        numResults=0;
+        subjectModel->notifyLayoutChanged();
+        predicateModel->notifyLayoutChanged();
+        predicateStatus->refreshAll();
+        objectModel->notifyLayoutChanged();
+        searchResultsModel->updateResultListChanged();
+        headerModel->updateDatasetChanged();
+        regexModel->updateDatasetChanged();
+
+        emit datasetChanged();
+    }
+}
+
+
+TripleComponentModel * HDTController::getSubjectModel()
+{
+    return subjectModel;
+}
+
+TripleComponentModel * HDTController::getPredicateModel()
+{
+    return predicateModel;
+}
+TripleComponentModel * HDTController::getObjectModel()
+{
+    return objectModel;
+}
+
+SearchResultsModel * HDTController::getSearchResultsModel()
+{
+    return searchResultsModel;
+}
+
+HeaderModel * HDTController::getHeaderModel()
+{
+    return headerModel;
+}
+
+RegexModel *HDTController::getRegexModel()
+{
+    return regexModel;
+}
+
+PredicateStatus *HDTController::getPredicateStatus()
+{
+    return predicateStatus;
+}
+
+HDTCachedInfo *HDTController::getHDTCachedInfo()
+{
+    return hdtCachedInfo;
+}
+
+hdt::HDT *HDTController::getHDT()
+{
+    return hdt;
+}
+
+bool HDTController::hasHDT()
+{
+    return hdt!=NULL;
+}
+
+void HDTController::setSearchPattern(hdt::TripleString &pattern)
+{
+    if(hdt==NULL) {
+        return;
+    }
+
+    if(searchPatternString != pattern) {
+        try {
+            this->searchPatternString = pattern;
+
+            hdt->getDictionary()->tripleStringtoTripleID(pattern, searchPatternID);
+
+            // Reset iterator
+            if(iteratorResults!=NULL) {
+                delete iteratorResults;
+                iteratorResults=NULL;
+            }
+            numResults=0;
+
+            // Check not found
+            if( (searchPatternID.getSubject()==0 && !pattern.getSubject().empty()) ||
+                    (searchPatternID.getPredicate()==0 && !pattern.getPredicate().empty()) ||
+                    (searchPatternID.getObject()==0 && !pattern.getObject().empty()) ) {
+
+            } else {
+                iteratorResults = hdt->getTriples()->search(searchPatternID);
+            }
+            updateNumResults();
+
+            predicateStatus->selectPredicate(searchPatternID.getPredicate());
+        } catch (char *exception){
+            numResults = 0;
+            iteratorResults = NULL;
+        } catch (const char *exception){
+            numResults = 0;
+            iteratorResults = NULL;
+        }
+        searchResultsModel->updateResultListChanged();
+
+        emit searchPatternChanged();
+    }
+}
+
+hdt::TripleID & HDTController::getSearchPatternID()
+{
+    return searchPatternID;
+}
+
+hdt::TripleString & HDTController::getSearchPatternString()
+{
+    return searchPatternString;
+}
+
+hdt::TripleID HDTController::getSelectedTriple()
+{
+    return selectedTriple;
+}
+
+void HDTController::setSelectedTriple(hdt::TripleID &selected)
+{
+    selectedTriple = selected;
+}
+
+void HDTController::clearSelectedTriple()
+{
+    selectedTriple.clear();
+}
+
+quint64 inline DIST(quint64 x1, quint64 x2, quint64 y1, quint64 y2) {
+        return (x1 - x2) * (x1 - x2) + (y1 - y2) * (y1 - y2);
+}
+
+void HDTController::selectNearestTriple(int subject, int predicate, int object)
+{
+    vector<hdt::TripleID> &triples = getTriples();
+    if(triples.size()==0) {
+        this->clearSelectedTriple();
+        return;
+    }
+
+    hdt::TripleID *best = &triples[0];
+    unsigned int bestpos = 0;
+    quint64 bestdist = DIST(triples[0].getSubject(), subject, triples[0].getObject(), object);
+
+    for (unsigned int i = 0; i < triples.size(); i ++) {
+        if (triples[i].match(searchPatternID) && predicateStatus->isPredicateActive(triples[i].getPredicate()-1)) {
+            quint64 dist = DIST(triples[i].getSubject(), subject, triples[i].getObject(), object);
+
+            if (dist < bestdist) {
+                best = &triples[i];
+                bestdist = dist;
+                bestpos = i;
+                //printf("1New %u, %u, %u, Dist: %u Pos: %u\n", best->getSubject(), best->getPredicate(), best->getObject(), bestdist, bestpos);
+            }
+        }
+    }
+    //printf("Found: %u, %u, %u, Dist: %llu\n", best->getSubject(), best->getPredicate(), best->getObject(), bestdist);
+
+    this->setSelectedTriple( *best );
+}
+
+vector<hdt::TripleID> & HDTController::getTriples()
+{
+    return hdtCachedInfo->getTriples();
+}
+
+size_t HDTController::getNumResults()
+{
+    return numResults;
+}
+
+void HDTController::numResultsValueChanged(int numResults)
+{
+    this->numResults = numResults;
+    searchResultsModel->updateNumResultsChanged();
+    emit numResultsChanged(numResults);
+}
+
+void HDTController:: numResultCountFinished()
+{
+
+}
+
+void HDTController::updateNumResults()
+{
+    if(iteratorResults==NULL) {
+        return;
+    }
+
+    if(iteratorResults->numResultEstimation()==hdt::EXACT) {
+        // Provided, use.
+        numResults = iteratorResults->estimatedNumResults();
+    } else {
+        // Not provided, count.
+        while(iteratorResults->hasNext()) {
+            iteratorResults->next();
+            numResults++;
+        }
+    }
+
+    searchResultsModel->updateNumResultsChanged();
+    emit numResultsChanged(numResults);
+}
+
+QString HDTController::getFileName()
+{
+    return fileName;
+}
+
+
+
+

package/deps/hdt-it/hdtcontroller.hpp

@@ -0,0 +1,103 @@
+#ifndef HDTMANAGER_HPP
+#define HDTMANAGER_HPP
+
+#include <HDT.hpp>
+#include <QObject>
+
+#include "hdtcachedinfo.hpp"
+#include "triplecomponentmodel.hpp"
+#include "searchresultsmodel.hpp"
+#include "headermodel.hpp"
+#include "predicatestatus.hpp"
+#include "regexmodel.hpp"
+
+class TripleComponentModel;
+class SearchResultsModel;
+class PredicateStatus;
+class HeaderModel;
+class RegexModel;
+
+class HDTController : public QObject
+{
+    Q_OBJECT
+
+public:
+    explicit HDTController(QObject *parent = 0);
+    ~HDTController();
+
+private:
+    // HDT
+    hdt::HDT *hdt;
+    QString fileName;
+
+    // Models / Info
+    TripleComponentModel *subjectModel;
+    TripleComponentModel *predicateModel;
+    TripleComponentModel *objectModel;
+    SearchResultsModel *searchResultsModel;
+    HeaderModel *headerModel;
+    PredicateStatus *predicateStatus;
+    HDTCachedInfo *hdtCachedInfo;
+    RegexModel *regexModel;
+
+    // Search pattern / results
+    hdt::TripleID selectedTriple;
+    hdt::TripleID searchPatternID;
+    hdt::TripleString searchPatternString;
+    hdt::IteratorTripleID *iteratorResults;
+    size_t numResults;
+
+public:
+    hdt::HDT *getHDT();
+    bool hasHDT();
+    void openHDTFile(QString file);
+    void saveHDTFile(QString file);
+    void importRDFFile(QString file, const string &baseUri, hdt::RDFNotation notation, hdt::HDTSpecification &spec);
+    void exportRDFFile(QString file, hdt::RDFNotation notation);
+    void exportResultsRDFFile(QString file, hdt::RDFNotation notation);
+    void closeHDT();
+
+    TripleComponentModel *getSubjectModel();
+    TripleComponentModel *getPredicateModel();
+    TripleComponentModel *getObjectModel();
+    RegexModel *getRegexModel();
+    SearchResultsModel *getSearchResultsModel();
+    HeaderModel *getHeaderModel();
+    PredicateStatus *getPredicateStatus();
+    HDTCachedInfo *getHDTCachedInfo();
+
+    void selectNearestTriple(int subject, int predicate, int object);
+    void setSelectedTriple(hdt::TripleID &selected);
+    hdt::TripleID getSelectedTriple();
+    void clearSelectedTriple();
+
+    hdt::TripleID &getSearchPatternID();
+    hdt::TripleString &getSearchPatternString();
+    void setSearchPattern(hdt::TripleString &pattern);
+
+    vector<hdt::TripleID> &getTriples();
+
+    size_t getNumResults();
+
+    QString getFileName();
+
+signals:
+    void datasetChanged();
+    void searchPatternChanged();
+    void numResultsChanged(int numResults);
+
+private slots:
+    void updateOnHDTChanged();
+    void updateNumResults();
+    void numResultsValueChanged(int numResults);
+    void numResultCountFinished();
+
+public slots:
+
+};
+
+
+#endif // HDTMANAGER_HPP
+
+
+

package/deps/hdt-it/hdtico.rc

@@ -0,0 +1 @@
+IDI_ICON1               ICON    DISCARDABLE     "hdtwinico2.ico"